mirror of
https://github.com/usestrix/strix.git
synced 2026-08-16 09:26:39 +02:00
Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e03133eddd | ||
|
|
0b45d223c3 | ||
|
|
230324d2b8 | ||
|
|
7d5a67d234 | ||
|
|
88ad3e4472 | ||
|
|
cf7689e927 | ||
|
|
3bb95ab43d | ||
|
|
9aa151c687 | ||
|
|
b9c2592b53 | ||
|
|
f54ecb74f9 | ||
|
|
96ca7e544d | ||
|
|
e4548cb28c | ||
|
|
df97c86f8f |
@@ -24,6 +24,7 @@ RUN apt-get update && \
|
||||
python3 python3-pip python3-dev python3-venv python3-setuptools \
|
||||
golang-go \
|
||||
net-tools dnsutils whois \
|
||||
file xxd \
|
||||
jq parallel ripgrep grep \
|
||||
less man-db procps htop \
|
||||
iproute2 iputils-ping netcat-traditional \
|
||||
@@ -192,6 +193,8 @@ RUN mkdir -p /workspace && chown -R pentester:pentester /workspace /app
|
||||
USER pentester
|
||||
RUN python3 -m venv /app/.venv && \
|
||||
/app/.venv/bin/pip install --no-cache-dir caido-sdk-client && \
|
||||
/app/.venv/bin/pip install --no-cache-dir \
|
||||
requests httpx beautifulsoup4 lxml pyjwt cryptography && \
|
||||
/app/.venv/bin/pip install --no-cache-dir -r /home/pentester/tools/jwt_tool/requirements.txt && \
|
||||
printf '%s\n' \
|
||||
'#!/bin/bash' \
|
||||
|
||||
@@ -91,10 +91,13 @@ http_proxy=http://127.0.0.1:${CAIDO_PORT}
|
||||
https_proxy=http://127.0.0.1:${CAIDO_PORT}
|
||||
EOF
|
||||
|
||||
echo "source /etc/profile.d/proxy.sh" >> ~/.bashrc
|
||||
echo "source /etc/profile.d/proxy.sh" >> ~/.zshrc
|
||||
# Use POSIX `.` (not the bashism `source`) so these lines are safe when the rc
|
||||
# files are read by a POSIX shell (e.g. `sh -lc`), which otherwise fails with
|
||||
# "source: not found". `.` is understood by bash, zsh, and dash alike.
|
||||
echo ". /etc/profile.d/proxy.sh" >> ~/.bashrc
|
||||
echo ". /etc/profile.d/proxy.sh" >> ~/.zshrc
|
||||
|
||||
source /etc/profile.d/proxy.sh
|
||||
. /etc/profile.d/proxy.sh
|
||||
|
||||
echo "✅ System-wide proxy configuration complete"
|
||||
|
||||
|
||||
@@ -81,6 +81,14 @@ Protocol-specific testing techniques.
|
||||
| --------- | ------------------------------------------------ |
|
||||
| `graphql` | GraphQL introspection, batching, resolver issues |
|
||||
|
||||
### Reconnaissance
|
||||
|
||||
Passive discovery and attack-surface mapping techniques.
|
||||
|
||||
| Skill | Coverage |
|
||||
| ----------------- | --------------------------------------------------------------- |
|
||||
| `asset_discovery` | CT, TLS SAN pivoting, passive DNS, and ASN/IP asset enumeration |
|
||||
|
||||
### Tooling
|
||||
|
||||
Sandbox CLI playbooks for core recon and scanning tools.
|
||||
|
||||
@@ -169,8 +169,23 @@ EFFICIENCY TACTICS:
|
||||
- Run multiple scans in parallel when possible
|
||||
- Load the most relevant skill before starting a specialized testing workflow if doing so will improve accuracy, speed, or tool usage
|
||||
- Use `exec_command` for Python code: write reusable scripts to a file and
|
||||
run them with `python3`. For one-off snippets, `python3 -c` or a
|
||||
here-document is acceptable.
|
||||
run them with `python3 script.py`. For one-off snippets, `python3 -c` or a
|
||||
here-document is acceptable, but avoid deeply nested quotes/parentheses — if
|
||||
a snippet needs complex quoting or is more than a few lines, write it to a
|
||||
file first to prevent syntax errors.
|
||||
- Before importing a third-party Python library, make sure it is installed. The
|
||||
sandbox's `python3` runs inside a preconfigured virtualenv that ships
|
||||
`requests`, `httpx`, `beautifulsoup4` (bs4), `lxml`, `pyjwt`, and
|
||||
`cryptography`; for anything else prefer the stdlib or run `pip install <pkg>`
|
||||
(it installs into that active venv) before importing, rather than letting the
|
||||
script fail with `ModuleNotFoundError`.
|
||||
- `exec_command` runs each command in a fresh non-interactive shell (plain
|
||||
pipes, no TTY). To drive an interactive or long-running process with
|
||||
`write_stdin` — REPLs, `ssh`/`nc`/`ftp`, `msfconsole`, or to send Ctrl-C —
|
||||
you MUST start it with `exec_command(cmd="...", tty=true)` and then
|
||||
`write_stdin(session_id=<id>, chars="...")`. Calling `write_stdin` on a
|
||||
default (non-TTY) command or on a process that has already exited fails with
|
||||
"stdin is not available".
|
||||
- For Caido proxy automation inside Python, explicitly import from
|
||||
`caido_api`:
|
||||
`from caido_api import list_requests, view_request, repeat_request, list_sitemap, view_sitemap_entry, scope_rules`
|
||||
@@ -186,7 +201,7 @@ EFFICIENCY TACTICS:
|
||||
VALIDATION REQUIREMENTS:
|
||||
- Full validation required - no assumptions
|
||||
- Demonstrate concrete impact with evidence
|
||||
- Consider business context for severity assessment
|
||||
- Consider business context for severity assessment — check whether the target is a demo/sandbox environment or content meant to be public, and factor that in
|
||||
- Independent verification through subagent
|
||||
- Document complete attack chain
|
||||
- Keep going until you find something that matters
|
||||
@@ -419,6 +434,7 @@ SPECIALIZED TOOLS:
|
||||
PROXY & INTERCEPTION:
|
||||
- Caido CLI - Modern web proxy (already running). Use the proxy tools
|
||||
directly, or import `caido_api` from sandbox Python scripts.
|
||||
- HTTPQL filters (for `list_requests`): quote string values, leave integers unquoted (`resp.code.eq:200`, not `"200"`); combine terms with `AND`/`OR` (there is no `NOT` — use the negated operator `ne`/`ncont`/`nregex`). Numeric fields (`resp.code`, `req.port`) use `eq`/`ne`/`gt`/`gte`/`lt`/`lte`; text fields (`req.host`, `req.path`, `req.method`, `req.raw`) use `cont`/`ncont`/`eq`/`regex`. Example: `resp.code.gte:200 AND resp.code.lt:300 AND req.host.cont:"api"`.
|
||||
- NOTE: If you are seeing proxy errors when sending requests, it usually means you are not sending requests to a correct url/host/port.
|
||||
- Ignore Caido proxy-generated 50x HTML error pages; these are proxy issues (might happen when requesting a wrong host or SSL/TLS issues, etc).
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from agents.models.multi_provider import MultiProvider
|
||||
from agents.retry import (
|
||||
ModelRetryBackoffSettings,
|
||||
ModelRetrySettings,
|
||||
RetryPolicyContext,
|
||||
retry_policies,
|
||||
)
|
||||
|
||||
@@ -20,6 +21,21 @@ if TYPE_CHECKING:
|
||||
from strix.config.settings import Settings
|
||||
|
||||
|
||||
def request_timeout_extra_args(timeout_s: float | None) -> dict[str, float] | None:
|
||||
"""Per-request model timeout; a plain float so ``ModelSettings.to_json_dict()`` stays serializable.""" # noqa: E501
|
||||
if not timeout_s or timeout_s <= 0:
|
||||
return None
|
||||
return {"timeout": timeout_s}
|
||||
|
||||
|
||||
def _retry_statusless_provider_errors(context: RetryPolicyContext) -> bool:
|
||||
"""Retry statusless provider errors (e.g. mid-stream quota/billing), but not aborts."""
|
||||
normalized = context.normalized
|
||||
if normalized.is_abort:
|
||||
return False
|
||||
return normalized.status_code is None
|
||||
|
||||
|
||||
class StrixProvider(MultiProvider):
|
||||
"""Route any non-OpenAI prefix through LiteLLM with the prefix preserved,
|
||||
so users type ``deepseek/deepseek-chat`` rather than
|
||||
@@ -56,6 +72,7 @@ DEFAULT_MODEL_RETRY = ModelRetrySettings(
|
||||
retry_policies.provider_suggested(),
|
||||
retry_policies.network_error(),
|
||||
retry_policies.http_status((429, 500, 502, 503, 504)),
|
||||
_retry_statusless_provider_errors,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
+1
-2
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from agents.lifecycle import RunHooks
|
||||
@@ -27,8 +28,6 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):
|
||||
"""Persist SDK-native usage after every model response."""
|
||||
|
||||
def __init__(self, *, model: str, max_budget_usd: float | None = None) -> None:
|
||||
import math
|
||||
|
||||
if max_budget_usd is not None and (
|
||||
not math.isfinite(max_budget_usd) or max_budget_usd <= 0
|
||||
):
|
||||
|
||||
@@ -12,6 +12,7 @@ from strix.config.models import (
|
||||
DEFAULT_MODEL_RETRY,
|
||||
is_known_openai_bare_model,
|
||||
model_supports_reasoning,
|
||||
request_timeout_extra_args,
|
||||
)
|
||||
from strix.core.sessions import scrub_images_from_items
|
||||
|
||||
@@ -126,11 +127,13 @@ def make_model_settings(
|
||||
*,
|
||||
model_name: str,
|
||||
force_required_tool_choice: bool = False,
|
||||
request_timeout: float | None = None,
|
||||
) -> ModelSettings:
|
||||
model_settings = ModelSettings(
|
||||
parallel_tool_calls=False,
|
||||
retry=DEFAULT_MODEL_RETRY,
|
||||
include_usage=True,
|
||||
extra_args=request_timeout_extra_args(request_timeout),
|
||||
)
|
||||
if (
|
||||
reasoning_effort is not None
|
||||
|
||||
@@ -215,6 +215,7 @@ async def run_strix_scan(
|
||||
settings.llm.reasoning_effort,
|
||||
model_name=resolved_model,
|
||||
force_required_tool_choice=settings.llm.force_required_tool_choice,
|
||||
request_timeout=settings.llm.timeout,
|
||||
)
|
||||
run_config = RunConfig(
|
||||
model=resolved_model,
|
||||
|
||||
@@ -16,6 +16,7 @@ from strix.config.models import (
|
||||
DEFAULT_MODEL_RETRY,
|
||||
StrixProvider,
|
||||
configure_sdk_model_defaults,
|
||||
request_timeout_extra_args,
|
||||
)
|
||||
from strix.report.state import get_global_report_state
|
||||
|
||||
@@ -310,7 +311,11 @@ async def check_duplicate(
|
||||
response = await model.get_response(
|
||||
system_instructions=DEDUPE_SYSTEM_PROMPT,
|
||||
input=user_msg,
|
||||
model_settings=ModelSettings(retry=DEFAULT_MODEL_RETRY, include_usage=True),
|
||||
model_settings=ModelSettings(
|
||||
retry=DEFAULT_MODEL_RETRY,
|
||||
include_usage=True,
|
||||
extra_args=request_timeout_extra_args(settings.llm.timeout),
|
||||
),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
|
||||
+102
-10
@@ -534,16 +534,10 @@ def litellm_cost_callback(
|
||||
cost = value
|
||||
|
||||
if cost is None:
|
||||
usage: Any = getattr(completion_response, "usage", None)
|
||||
if usage is None and isinstance(completion_response, dict):
|
||||
usage = cast("dict[str, Any]", completion_response).get("usage")
|
||||
usage_cost: Any
|
||||
if isinstance(usage, dict):
|
||||
usage_cost = cast("dict[str, Any]", usage).get("cost")
|
||||
else:
|
||||
usage_cost = getattr(usage, "cost", None)
|
||||
if isinstance(usage_cost, int | float) and usage_cost > 0:
|
||||
cost = float(usage_cost)
|
||||
cost = _usage_reported_cost(completion_response)
|
||||
|
||||
if cost is None:
|
||||
cost = _estimate_response_cost(kwargs, completion_response)
|
||||
|
||||
if cost is None or cost <= 0:
|
||||
return
|
||||
@@ -554,3 +548,101 @@ def litellm_cost_callback(
|
||||
report_state.record_observed_llm_cost(cost)
|
||||
except Exception:
|
||||
logger.exception("Failed to record observed LiteLLM cost")
|
||||
|
||||
|
||||
def _usage_reported_cost(completion_response: Any) -> float | None:
|
||||
"""Provider-reported cost from the ``usage`` block (e.g. OpenRouter).
|
||||
|
||||
Non-BYOK responses charge everything to ``usage.cost``. BYOK responses
|
||||
charge only the OpenRouter fee to ``usage.cost`` (often 0) and report the
|
||||
provider charge in ``usage.cost_details.upstream_inference_cost``, so the
|
||||
true BYOK total is the sum of the two.
|
||||
"""
|
||||
usage: Any = getattr(completion_response, "usage", None)
|
||||
if usage is None and isinstance(completion_response, dict):
|
||||
usage = cast("dict[str, Any]", completion_response).get("usage")
|
||||
if usage is None:
|
||||
return None
|
||||
|
||||
def _field(container: Any, name: str) -> Any:
|
||||
if isinstance(container, dict):
|
||||
return cast("dict[str, Any]", container).get(name)
|
||||
return getattr(container, name, None)
|
||||
|
||||
total = 0.0
|
||||
usage_cost = _field(usage, "cost")
|
||||
if isinstance(usage_cost, int | float) and usage_cost > 0:
|
||||
total += float(usage_cost)
|
||||
|
||||
if bool(_field(usage, "is_byok")):
|
||||
upstream = _field(_field(usage, "cost_details"), "upstream_inference_cost")
|
||||
if isinstance(upstream, int | float) and upstream > 0:
|
||||
total += float(upstream)
|
||||
|
||||
return total if total > 0 else None
|
||||
|
||||
|
||||
def _estimate_response_cost(kwargs: Any, completion_response: Any) -> float | None:
|
||||
"""Best-effort LiteLLM cost-map estimate when no provider-reported cost exists.
|
||||
|
||||
LiteLLM strips provider cost fields when rebuilding streamed responses and
|
||||
returns no ``response_cost`` for models missing from its cost map, so try
|
||||
the provider-prefixed name, the raw name, and the bare model name.
|
||||
"""
|
||||
from litellm import completion_cost
|
||||
|
||||
model = kwargs.get("model") if isinstance(kwargs, dict) else None
|
||||
if not isinstance(model, str) or not model:
|
||||
if isinstance(completion_response, dict):
|
||||
model = cast("dict[str, Any]", completion_response).get("model")
|
||||
else:
|
||||
model = getattr(completion_response, "model", None)
|
||||
if not isinstance(model, str) or not model:
|
||||
return None
|
||||
|
||||
provider = None
|
||||
litellm_params = kwargs.get("litellm_params") if isinstance(kwargs, dict) else None
|
||||
if isinstance(litellm_params, dict):
|
||||
provider = litellm_params.get("custom_llm_provider")
|
||||
|
||||
usage_payload = _usage_payload(completion_response)
|
||||
if usage_payload is None:
|
||||
return None
|
||||
|
||||
candidates: list[str] = []
|
||||
if isinstance(provider, str) and provider and not model.startswith(f"{provider}/"):
|
||||
candidates.append(f"{provider}/{model}")
|
||||
candidates.append(model)
|
||||
if "/" in model:
|
||||
candidates.append(model.rsplit("/", 1)[-1])
|
||||
|
||||
for candidate in candidates:
|
||||
try:
|
||||
value = completion_cost(
|
||||
completion_response={"model": candidate, "usage": usage_payload},
|
||||
model=candidate,
|
||||
)
|
||||
except Exception: # nosec B112 # noqa: BLE001, S112
|
||||
continue
|
||||
if isinstance(value, int | float) and value > 0:
|
||||
return float(value)
|
||||
return None
|
||||
|
||||
|
||||
def _usage_payload(completion_response: Any) -> dict[str, Any] | None:
|
||||
"""Token counts as a plain dict, detached from the response's provider metadata."""
|
||||
usage: Any = getattr(completion_response, "usage", None)
|
||||
if usage is None and isinstance(completion_response, dict):
|
||||
usage = cast("dict[str, Any]", completion_response).get("usage")
|
||||
if usage is None:
|
||||
return None
|
||||
if hasattr(usage, "model_dump"):
|
||||
usage = usage.model_dump()
|
||||
if not isinstance(usage, dict):
|
||||
return None
|
||||
payload = cast("dict[str, Any]", usage)
|
||||
if not payload.get("total_tokens") and not (
|
||||
payload.get("prompt_tokens") or payload.get("completion_tokens")
|
||||
):
|
||||
return None
|
||||
return payload
|
||||
|
||||
@@ -10,6 +10,7 @@ exposed-port URL for all subsequent SDK calls.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -93,9 +94,16 @@ async def bootstrap_caido(
|
||||
client = Client(host_url, auth=TokenAuthOptions(token=access_token))
|
||||
await client.connect()
|
||||
|
||||
project = await client.project.create(
|
||||
CreateProjectOptions(name="sandbox", temporary=True),
|
||||
)
|
||||
await client.project.select(project.id)
|
||||
try:
|
||||
project = await client.project.create(
|
||||
CreateProjectOptions(name="sandbox", temporary=True),
|
||||
)
|
||||
await client.project.select(project.id)
|
||||
except BaseException:
|
||||
# The connected client never reaches the session bundle if project
|
||||
# setup fails, so close it here to avoid leaking the transport.
|
||||
with contextlib.suppress(Exception):
|
||||
await client.aclose()
|
||||
raise
|
||||
logger.info("Caido project selected: %s", project.id)
|
||||
return client
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
---
|
||||
name: asset-discovery
|
||||
description: Passive asset and attack-surface discovery via certificate transparency, TLS SAN pivoting, passive DNS, and ASN/IP enumeration to find hosts beyond subdomain brute force
|
||||
---
|
||||
|
||||
# Asset Discovery
|
||||
|
||||
Most engagements start from a small seed (one domain, one org name) but the real attack surface is far larger: forgotten hosts, staging/internal-named services, acquisitions, and infrastructure that never appears in a wordlist. Build a broad, deduplicated inventory using passive intelligence — certificate transparency, TLS certificate metadata, passive DNS, and ASN/IP data — then collapse it into a probed, classified attack surface. The aim is coverage and pivoting: every certificate, DNS record, and IP is a lead to more assets.
|
||||
|
||||
Only use this skill when all subdomains and related assets of the target are in scope — broad discovery pulls in hosts far beyond the seed.
|
||||
|
||||
## Attack Surface
|
||||
|
||||
- Hosts discoverable via issued certificates (CT logs) but absent from DNS brute force
|
||||
- Internal/staging/pre-prod hostnames leaked in certificate SAN lists
|
||||
- Sibling and acquisition domains sharing certificates, ASNs, or IP ranges with the seed
|
||||
- Wildcard and short-lived certs revealing naming conventions (`*.internal.example.com`, `k8s-*`, `argocd.*`)
|
||||
- ASN-owned IP ranges hosting services with no DNS name at all
|
||||
- Virtual hosts co-located on shared IPs (multiple apps behind one address)
|
||||
- Non-HTTP services on discovered hosts (databases, brokers, admin ports)
|
||||
|
||||
## High-Value Sources
|
||||
|
||||
### Certificate Transparency (CT)
|
||||
|
||||
CT logs record nearly every publicly-trusted certificate. Query by domain (matches SAN/CN) and by organization name.
|
||||
|
||||
- **crt.sh** (free, no key):
|
||||
- By domain incl. subdomains: `curl -s 'https://crt.sh/?q=%25.example.com&output=json' | jq -r '.[].name_value' | sed 's/^\*\.//' | sort -u`
|
||||
- By organization: `https://crt.sh/?O=Example+Inc&output=json`
|
||||
- **Censys / Shodan / Fofa** (API keys): search certs by `parsed.names`, `parsed.subject.organization`, or a specific `fingerprint_sha256`, then pivot to every host serving that cert.
|
||||
- Cross-check multiple indexes (`certspotter`, Google CT, `chaos`) — no single log is complete.
|
||||
- **Wildcards** (`*.corp.example.com`) reveal internal naming schemes even when individual hosts resolve privately; use them to seed targeted guesses (`grafana.corp`, `ci.corp`, `vault.corp`).
|
||||
|
||||
### TLS Certificate SAN/CN
|
||||
|
||||
- **SAN expansion**: one cert often lists many hostnames (marketing + api + admin + internal) — extract every SAN, not just the queried name.
|
||||
- **Shared-cert pivot**: the same cert fingerprint served on multiple IPs ties disparate assets to one owner.
|
||||
- **Issuer/org pivot**: certs sharing `subject.organization`/`organizationalUnit` frequently belong to the same target.
|
||||
- **Active read** catches names never submitted to public CT: `echo | openssl s_client -connect HOST:443 -servername HOST 2>/dev/null | openssl x509 -noout -text | grep -A1 'Subject Alternative Name'`
|
||||
- **Internal leak signal**: SANs like `localhost`, `*.internal`, `*.svc.cluster.local`, `*.local`, or RFC1918-style names on a public cert expose internal naming and sometimes internal services fronted publicly.
|
||||
|
||||
### Passive DNS
|
||||
|
||||
- Forward-resolve every name (A/AAAA/CNAME); keep CNAME chains — they reveal third-party providers and CDNs.
|
||||
- **Reverse DNS (PTR)** on discovered IPs surfaces co-located hostnames.
|
||||
- **Historical/passive DNS** (SecurityTrails, VirusTotal, `chaos`, passivedns providers) recovers names that no longer resolve but may still front live infra.
|
||||
|
||||
### ASN & IP Ranges
|
||||
|
||||
- Map a known IP to its ASN and netblock: `whois -h whois.cymru.com " -v <IP>"` or a BGP/ASN lookup.
|
||||
- If the org runs its own ASN, enumerate all announced prefixes and treat them as candidate assets.
|
||||
- For cloud-hosted targets the IP belongs to the provider, not the org — pivot via cert/vhost instead of netblock.
|
||||
|
||||
## Recommended Tooling
|
||||
|
||||
Prefer the projectdiscovery suite (already available in the sandbox and pipeline-friendly with JSON output):
|
||||
|
||||
- **`subfinder`** — passive subdomain aggregation across many sources incl. CT: `subfinder -d example.com -all -recursive -silent -oJ -o subs.jsonl`
|
||||
- **`tlsx`** — TLS/cert data at scale; grab SANs and issuer/org to pivot: `tlsx -l hosts.txt -san -cn -tls-version -json -o tls.jsonl`
|
||||
- **`uncover`** — query Shodan/Censys/Fofa/Quake/crt.sh engines from one CLI: `uncover -q 'ssl:"Example Inc"' -e shodan,censys,fofa -json`
|
||||
- **`asnmap`** — org/domain/ASN → CIDR ranges: `asnmap -d example.com -json` / `asnmap -org "Example Inc"`
|
||||
- **`mapcidr`** — expand/aggregate CIDRs into host lists for probing: `mapcidr -cidr 192.0.2.0/24 -o hosts.txt`
|
||||
- **`dnsx`** — fast resolution, PTR, and wildcard filtering: `dnsx -l names.txt -a -aaaa -cname -ptr -resp -json -o dns.jsonl`
|
||||
- **`httpx`** — live probing + cert grab in one pass (see methodology).
|
||||
- **`naabu`** — port sweep for non-HTTP services: `naabu -list hosts.txt -top-ports 100 -verify -silent`
|
||||
|
||||
Also useful: **`amass`** (`amass intel`/`enum` for ASN, cert, and passive sources), **`cero`** (bulk SAN extraction from IPs/ranges), and direct **crt.sh** JSON queries when no keys are configured. Cross-source results — CT + passive DNS + `subfinder` together beat any single source.
|
||||
|
||||
## Key Techniques
|
||||
|
||||
### Iterative Seed Expansion
|
||||
|
||||
Every new name, PTR result, CNAME target, and cert SAN becomes a fresh seed. Loop CT → SAN extraction → passive DNS → ASN/range expansion until the asset set stops growing.
|
||||
|
||||
### Cert-Fingerprint Pivoting
|
||||
|
||||
Search Censys/Shodan (or `uncover`) by a cert's `fingerprint_sha256` to find every other host presenting the same certificate — the strongest cross-asset link for tying acquisitions and shadow infra to the target.
|
||||
|
||||
### Naming-Convention Inference
|
||||
|
||||
Wildcard SANs and observed hostnames expose the org's naming scheme; generate targeted candidates from it (`<service>.<env>.example.com`) rather than blind brute force.
|
||||
|
||||
### IP-First Discovery
|
||||
|
||||
For ASN-owned ranges, sweep IPs directly with `naabu`/`httpx` and read served certs (`tlsx`) to find services that have no DNS name at all.
|
||||
|
||||
## Advanced Techniques
|
||||
|
||||
- **Active SAN harvesting** across whole ranges with `tlsx`/`cero` recovers internal hostnames never logged to public CT.
|
||||
- **Favicon and response hashing** (`httpx -favicon`, hash pivots in Shodan) clusters instances of the same app across unrelated hostnames.
|
||||
- **Vhost differentials**: probe a single IP with multiple `Host:` values to unmask co-located apps behind one address.
|
||||
- **Historical CT/DNS diffing** highlights recently issued certs and newly appearing hosts — high-signal for fresh or misconfigured deployments.
|
||||
|
||||
## Consolidation & Probing
|
||||
|
||||
1. **Dedupe** names and IPs into one inventory; record source(s) per asset for confidence.
|
||||
2. **Live probe** with `httpx`, capturing status/title/tech/server and cert SANs in one pass — each grabbed SAN feeds back as a new seed:
|
||||
`httpx -l hosts.txt -sc -title -server -td -tls-grab -json -o assets.jsonl`
|
||||
3. **Classify** assets by function from title/tech/path signals: app, API, marketing, auth, CI/CD, observability, storage, admin, VCS, mail. Cluster by role, not by a specific product.
|
||||
4. **Port sweep** interesting hosts with `naabu` for non-HTTP services (DBs, caches, brokers, mgmt ports).
|
||||
5. **Prioritize** by exposure and value, then hand each finding to the right specialist skill:
|
||||
- Exposed dashboards / debug / observability / metadata leaks → `information_disclosure`
|
||||
- Login/admin panels with default or weak creds → `weak_password_detection`
|
||||
- Dangling DNS / unclaimed provider resources → `subdomain_takeover`
|
||||
- Cloud consoles/metadata surfaces → `aws` / `gcp` / `kubernetes`
|
||||
|
||||
## Testing Methodology
|
||||
|
||||
1. **Seed** - domains, org/legal names, known IPs, email domains, code-host org
|
||||
2. **Certificate transparency** - pull all logged certs per seed domain and org name (crt.sh, `uncover`)
|
||||
3. **SAN/CN extraction** - parse every Subject CN and SAN with `tlsx`; each new name is a new seed
|
||||
4. **Passive DNS** - resolve forward and reverse with `dnsx`; harvest historical records
|
||||
5. **ASN/IP mapping** - `asnmap` → `mapcidr` to expand owned ranges, then sweep for live hosts
|
||||
6. **Active TLS pivot** - `tlsx`/`cero` on live IPs/ports to grab SANs missing from public CT
|
||||
7. **Consolidate & probe** - dedupe, `httpx` probe, classify, and route to specialists
|
||||
|
||||
## Validation
|
||||
|
||||
1. Confirm each discovered asset actually resolves and serves content (live `httpx` result, not just a passive hit)
|
||||
2. Attribute assets to the target via matching cert org, shared cert fingerprint, or DNS under a seed domain
|
||||
3. Deduplicate vhost aliases and CDN edges down to distinct origins so the surface is not inflated
|
||||
4. Record provenance (which source produced each asset) for reproducibility
|
||||
|
||||
## False Positives
|
||||
|
||||
- CDN/edge hostnames and provider default names that are not org-owned
|
||||
- Shared-hosting neighbors on the same IP (vhost co-tenancy, not the target's asset)
|
||||
- Stale historical DNS entries pointing at reassigned infrastructure
|
||||
- Wildcard-cert-implied hostnames that never actually resolve or serve content
|
||||
|
||||
## Impact
|
||||
|
||||
- Expanded attack surface: forgotten, staging, and internal-named hosts brute force misses
|
||||
- Discovery of misconfigured or unauthenticated services fronted by leaked internal hostnames
|
||||
- Attribution of shadow infra, acquisitions, and sibling domains to the target
|
||||
- A prioritized, classified inventory that feeds every downstream specialist skill
|
||||
|
||||
## Pro Tips
|
||||
|
||||
1. Loop the pipeline — every SAN, PTR, and CNAME target is a new seed until the set converges.
|
||||
2. crt.sh is the cheapest high-yield source (no key); Censys/Shodan via `uncover` add cert-fingerprint and vhost pivoting when keys exist.
|
||||
3. Always cert-grab live hosts with `tlsx` — active SANs catch internal hostnames never sent to public CT.
|
||||
4. Internal-looking SANs (`*.internal`, `*.svc.cluster.local`, staging names) are the highest-signal leads.
|
||||
5. Wildcard SANs reveal naming conventions — seed targeted guesses instead of blind brute force.
|
||||
6. Cluster by function, not product name, so the workflow generalizes to any exposed service.
|
||||
7. Keep JSON output throughout so stages chain cleanly (`subfinder` → `dnsx` → `httpx` → `naabu`).
|
||||
|
||||
## Summary
|
||||
|
||||
Broad passive discovery — CT + TLS SAN pivoting + passive DNS + ASN/IP mapping, looped until convergence — finds the assets brute force misses, especially internal-named and forgotten services leaked through certificates. Build the inventory with the projectdiscovery suite, probe and classify it generically, then route each interesting asset to the specialist skill for its class.
|
||||
@@ -0,0 +1,189 @@
|
||||
---
|
||||
name: grafana_prometheus
|
||||
description: Grafana, Prometheus, Alertmanager and exporter security testing — turning exposed observability into SSRF, credential theft, RCE, and lateral movement into the internal network
|
||||
---
|
||||
|
||||
# Grafana & Prometheus (Observability Stack)
|
||||
|
||||
Observability stacks (Grafana + Prometheus + Alertmanager + Loki/Tempo/Jaeger + exporters) are among the highest-value pivots on a network. They are chronically exposed (300k+ internet-facing Grafana instances on Shodan), run with weak/no auth, hold plaintext credentials for every backend they touch, and sit in a network position that reaches internal services and cloud metadata. Treat a reachable observability endpoint not as the finding but as the **entry point**: the goal is to pivot from "monitoring is exposed" into data-source credential theft, SSRF into the internal network, cloud key compromise, RCE, and cluster/host takeover.
|
||||
|
||||
## Attack Surface
|
||||
|
||||
**Grafana** (default `:3000`)
|
||||
- Web UI + REST API (`/api/*`), login, org/user management, snapshots
|
||||
- Data sources: stored connection details + credentials for Prometheus, Loki, Tempo, MySQL/Postgres, Elasticsearch, InfluxDB, CloudWatch, Azure Monitor, etc.
|
||||
- Data source **proxy** (`/api/datasources/proxy/...`, `/api/ds/query`) — server-side HTTP client → SSRF primitive
|
||||
- Plugins (incl. Image Renderer, Infinity) — extra SSRF/RCE surface
|
||||
- Alerting → contact points/webhooks (outbound HTTP, another SSRF vector)
|
||||
|
||||
**Prometheus** (default `:9090`)
|
||||
- Query API (`/api/v1/query`, `/graph`), config/target/status endpoints, federation, admin/lifecycle API
|
||||
|
||||
**Alertmanager** (default `:9093`)
|
||||
- Alert/silence API (`/api/v2/*`), config with receiver credentials
|
||||
|
||||
**Exporters / adjacent** — node_exporter (`:9100`), cAdvisor/kubelet (`:4194`/`:10250`), kube-state-metrics (`:8080`), Pushgateway (`:9091`), Loki (`:3100`), Tempo, Jaeger UI (`:16686`), Thanos/Cortex/Mimir/VictoriaMetrics
|
||||
|
||||
## Reconnaissance
|
||||
|
||||
**Fingerprint & version** (version drives which CVEs apply)
|
||||
```
|
||||
GET /api/health # Grafana: {"version":"...","commit":"..."}
|
||||
GET /api/frontend/settings # buildInfo, enabled auth, datasource types
|
||||
GET /login # Grafana login page / footer version
|
||||
GET /api/v1/status/buildinfo # Prometheus version
|
||||
GET /metrics # any exporter → prometheus/node/go_* series
|
||||
```
|
||||
|
||||
**Auth posture — always test unauthenticated first**
|
||||
```
|
||||
GET /api/datasources # Grafana: 200 = anon/viewer has admin-ish read
|
||||
GET /?orgId=1 # anonymous access enabled? lands on dashboards
|
||||
GET /api/v1/targets # Prometheus: 200 = no auth
|
||||
GET /api/v2/status # Alertmanager: 200 = no auth
|
||||
```
|
||||
|
||||
**Credential entry points**
|
||||
- Grafana default creds `admin:admin` (the first-login change prompt has a **Skip** button — ~1 in 5 internet-facing instances still accept it)
|
||||
- Anonymous org access (`auth.anonymous`), open sign-up, guest/viewer roles
|
||||
- Leaked Grafana API keys / service account tokens (`Authorization: Bearer glsa_...` / `eyJ...`) in JS bundles, git, CI logs
|
||||
|
||||
## Key Vulnerabilities & CVEs
|
||||
|
||||
### CVE-2021-43798 — Grafana pre-auth path traversal (arbitrary file read)
|
||||
Grafana 8.0.0-beta1 → 8.3.0. Directory traversal through the plugin static route reads any file the process can, **no auth required**. Every install ships pre-installed plugins, so the path always exists.
|
||||
```
|
||||
curl --path-as-is 'http://host:3000/public/plugins/mysql/../../../../../../../../etc/passwd'
|
||||
# other plugin ids that always exist: prometheus, graph, text, alertlist, table-old
|
||||
```
|
||||
High-value reads:
|
||||
- `/etc/grafana/grafana.ini` and `conf/defaults.ini` → `secret_key`, admin password, SMTP/LDAP creds
|
||||
- `/var/lib/grafana/grafana.db` (SQLite) → `data_source.secure_json_data` (AES-encrypted with `secret_key` → decrypt to recover backend passwords/tokens), session tokens, API key hashes
|
||||
- `/proc/self/environ`, cloud credential files (`~/.aws/credentials`, k8s SA token at `/var/run/secrets/kubernetes.io/serviceaccount/token`)
|
||||
|
||||
### CVE-2024-9264 — Grafana SQL Expressions RCE + LFI (DuckDB)
|
||||
Grafana **v11.0.0–11.2.x** (10.x not affected). The experimental SQL Expressions feature passes user input to the `duckdb` CLI insufficiently sanitized → command injection + arbitrary file read. Enabled by default for the API (feature-flag bug); exploitable **only if the `duckdb` binary is in Grafana's `$PATH`** (not shipped by default). Any user with **Viewer or higher** can exploit. CVSS 9.4.
|
||||
- Probe: is `duckdb` present? Try the SQL Expressions query path; LFI via `read_csv`/`read_blob`-style functions, command injection via DuckDB's shell/`install`/`load` extension mechanics.
|
||||
- Mitigation you'll see: remove `duckdb` from PATH.
|
||||
|
||||
### CVE-2025-4123 — Grafana open redirect + stored XSS → SSRF chain
|
||||
Double-encoded traversal (`..%2f`) into the client path/`/redirect` forwards the victim to an attacker origin that serves a malicious plugin manifest → JS executes in the trusted grafana origin (stored XSS). If the **Image Renderer** plugin is present, escalate to full-read SSRF:
|
||||
```
|
||||
POST /api/render?url=http://169.254.169.254/latest/meta-data/iam/security-credentials/
|
||||
```
|
||||
No creds needed when anonymous access is on (common in demo/lab).
|
||||
|
||||
### CVE-2021-39226 / CVE-2024-1313 — Grafana snapshot auth bypass
|
||||
Unauthenticated view (and, with `public_mode`, delete) of the lowest-key snapshot via `/api/snapshots/:key` and `/dashboard/snapshot/:key`; CVE-2024-1313 lets a user in a *different org* delete snapshots by view key. Walk snapshot IDs to harvest dashboard data / leaked query values.
|
||||
|
||||
### Prometheus / Alertmanager — exposure is the vuln (no auth by default)
|
||||
Prometheus and Alertmanager ship with **no authentication**; the docs explicitly say do not expose them. There is rarely a CVE — reachability itself is the finding, and the payoff is recon + credential leakage + pivoting (below).
|
||||
|
||||
## Pivoting: Observability → Deeper Compromise
|
||||
|
||||
This is the core value. Chain each exposure into something that matters. Always articulate the pivot in the finding, not just the exposed endpoint.
|
||||
|
||||
### 1. Grafana data-source proxy → full-read SSRF (internal net + cloud metadata)
|
||||
Grafana OSS ships a **no-op URL validator** and an **empty `data_source_proxy_whitelist`** (empty = allow all). The proxy resolves the proxied path against the **selected data source's configured base URL**, so to reach an arbitrary host you must first create (or edit) a data source whose URL is the internal/metadata target — this needs data-source write permission (Editor/Admin, or any role granted `datasources:create`/`:write`). Reusing an ordinary Prometheus data-source id and appending a metadata path just hits Prometheus, not the metadata service — do not report that as SSRF. Once a data source points at the target, the proxy issues the request server-side and returns the **full response body**.
|
||||
```
|
||||
# Step 1: create/edit a data source with an attacker-chosen base URL, e.g.
|
||||
POST /api/datasources {"name":"x","type":"prometheus","access":"proxy",
|
||||
"url":"http://169.254.169.254"} # returns the new <id>
|
||||
# Step 2: relay through THAT data source's id (path appended to its base URL):
|
||||
GET /api/datasources/proxy/<id>/latest/meta-data/iam/security-credentials/<role> # AWS IMDSv1
|
||||
# GCP: base url http://metadata.google.internal + header Metadata-Flavor: Google
|
||||
# → /computeMetadata/v1/instance/service-accounts/default/token
|
||||
# Internal APIs, k8s API server, admin panels, other cloud services (one DS per host)
|
||||
```
|
||||
Pivot: metadata creds → cloud account; internal API reads → data; network mapping → next target. Also test the **alerting contact-point/webhook** (attacker-controlled outbound URL) and plugin SSRFs (e.g. Infinity CVE-2025-8341) as independent vectors. The **Image Renderer** is an SSRF vector too, but not via an arbitrary-URL proxy: it renders Grafana dashboard/panel render routes (`/render/d-solo/...`), so the SSRF arises when a render request is coerced to fetch an internal URL (e.g. chained with CVE-2025-4123), not from a `?url=` parameter.
|
||||
|
||||
### 2. Grafana admin → harvest every backend credential
|
||||
Once authenticated (default creds, anon-admin, leaked token, or after CVE-2021-43798):
|
||||
```
|
||||
GET /api/datasources # host, port, db, user for 5–15 backends
|
||||
GET /api/admin/settings # SMTP, LDAP bind, OAuth secrets, DB DSN (grafana.ini runtime)
|
||||
```
|
||||
Grafana stores backend passwords/tokens encrypted (`secureJsonData`) — the API won't echo them, but you can (a) use the data source proxy to **query the backend directly through Grafana** (no plaintext needed), or (b) decrypt `grafana.db` `secure_json_data` with the leaked `secret_key` (from grafana.ini) offline. Each recovered credential (Postgres, MySQL, Elasticsearch, CloudWatch/Azure keys) is a fresh pivot into that system.
|
||||
|
||||
### 3. Prometheus config/targets → leaked scrape credentials + inventory
|
||||
```
|
||||
GET /api/v1/status/config # loaded prometheus.yml
|
||||
GET /api/v1/targets # every scrape target + discovery metadata labels
|
||||
```
|
||||
Prometheus renders secret-typed fields (`basic_auth.password`, `authorization.credentials`, bearer tokens, OAuth client secrets — including inside `remote_write`/`remote_read`) as `<secret>` in the config response, so do **not** report those as leaked unless the actual value is shown. What genuinely leaks: **usernames** (`basic_auth.username`), and — critically — **credentials embedded in target/endpoint URLs** (`https://user:pass@host/...`), which are *not* masked. `remote_write`/`remote_read` blocks still reveal internal backend endpoints (Grafana Cloud/Cortex/Mimir/Thanos hosts) and usernames even with secrets redacted. `kubernetes_sd_configs` and cloud SD expose internal DNS and can surface creds via URL fields. Target lists + `__meta_*`/`__address__` labels = a free internal network map (hostnames, ports, k8s namespaces, cloud instance IDs).
|
||||
|
||||
### 4. PromQL / metrics → internal topology, versions → known-CVE targeting
|
||||
Metrics are a recon goldmine. Query without auth:
|
||||
```
|
||||
GET /api/v1/query?query=up # every monitored service (host:port)
|
||||
GET /api/v1/query?query=node_uname_info # kernel/OS/host
|
||||
GET /api/v1/query?query=node_dmi_info # cloud provider / hardware
|
||||
GET /api/v1/query?query=node_network_info # interfaces, internal IPs/MACs
|
||||
GET /api/v1/query?query=kube_pod_info # pods, namespaces, node IPs (KSM)
|
||||
GET /api/v1/query?query=kube_node_info # node hostnames, kubelet/kubeproxy versions
|
||||
GET /api/v1/query?query={__name__=~"..._build_info"} # exact component versions
|
||||
GET /api/v1/label/__name__/values # enumerate all metric names → app inventory
|
||||
GET /federate?match[]={__name__=~".%2b"} # bulk-exfil series via federation
|
||||
```
|
||||
Pivot: exact versions (`*_build_info`, `kube_node_info`) → map to CVEs and attack the vulnerable components; `up`/`kube_pod_info` → target list of internal services normally invisible from outside. cAdvisor/kubelet and kube-state-metrics reveal container images, args, labels (sometimes secrets in env-derived labels), and full cluster layout.
|
||||
|
||||
### 5. Alertmanager → credential theft, SSRF, and alert suppression (anti-forensics)
|
||||
```
|
||||
GET /api/v2/status # config (receiver creds often masked, structure/routes leak)
|
||||
POST /api/v2/silences # unauth in default deploys → silence ALL alerts
|
||||
```
|
||||
- Receiver config (`alertmanager.yml`) holds **plaintext** Slack webhook URLs, PagerDuty routing keys, SMTP passwords, OpsGenie/VictorOps keys — steal via file read (CVE-2021-43798 style) or config access; reuse to spoof alerts / social-engineer on-call.
|
||||
- Webhook receivers = SSRF: if you can influence the receiver URL, point it at internal endpoints.
|
||||
- Silence abuse: `POST /api/v2/silences` with matcher `alertname=~".+"` for 30d suppresses security/ops alerting while you operate — call this out as a **detection-evasion** impact.
|
||||
|
||||
### 6. Logs/traces backends (Loki, Tempo, Jaeger) → secrets in transit
|
||||
Exposed Loki (`/loki/api/v1/query_range`), Tempo, and Jaeger UI (`:16686`) frequently contain **request bodies, headers, tokens, session cookies, SQL, and stack traces** captured from real traffic. Query them for `authorization`, `password`, `token`, `set-cookie`, PII. A single logged bearer token or session cookie is a direct account/service takeover.
|
||||
|
||||
## Testing Methodology
|
||||
|
||||
1. **Discover** stack ports/services (`:3000/:9090/:9093/:9100/:3100/:16686`, `/metrics`, `/api/health`).
|
||||
2. **Fingerprint versions** → shortlist applicable CVEs (43798, 9264, 4123, 39226/1313, Infinity 8341).
|
||||
3. **Auth matrix** — unauth vs anon vs viewer vs default creds vs leaked token, per component.
|
||||
4. **Recon-pivot** — pull Prometheus config/targets + PromQL inventory; enumerate Grafana `/api/datasources`.
|
||||
5. **SSRF-pivot** — data source proxy / render / webhook → internal services + `169.254.169.254`.
|
||||
6. **Credential-pivot** — file read (43798) → `secret_key` → decrypt `grafana.db`; scrape/remote_write/receiver creds; then reuse against each backend.
|
||||
7. **Deepen** — RCE (9264 if `duckdb` present), cloud account via metadata, k8s SA token, DB access; demonstrate real impact.
|
||||
|
||||
## Validation
|
||||
|
||||
- SSRF: show the **full body** of an internal-only URL (metadata creds, internal API JSON) returned through Grafana — not just a timing/blind signal.
|
||||
- Credential theft: show the leaked secret AND prove reuse (authenticate to the backend / cloud), or clearly explain the reuse path.
|
||||
- File read (43798): return contents of `/etc/passwd` or `grafana.ini` with `--path-as-is`; note affected version.
|
||||
- RCE (9264): confirm `duckdb` in PATH first; demonstrate command execution or file read; note version 11.x.
|
||||
- Recon: for Prometheus/Alertmanager exposure, pair the open endpoint with the concrete sensitive data recovered (leaked creds, internal inventory) so the finding shows impact, not just "it's reachable".
|
||||
|
||||
## False Positives / Down-rate
|
||||
|
||||
- Endpoint reachable only from localhost / same trusted segment by design, behind an authenticating reverse proxy (test through the real ingress).
|
||||
- Grafana Enterprise (real URL validator) or OSS with a configured `data_source_proxy_whitelist` → SSRF blocked.
|
||||
- CVE-2024-9264 with **no `duckdb` in PATH** → not exploitable (do not report as RCE).
|
||||
- Patched versions (Grafana ≥ the fixed release for each CVE; check `/api/health`).
|
||||
- **Demo/sandbox instances with synthetic data** — down-rate per demo-data guidance; exposed monitoring of a throwaway target is low impact.
|
||||
- Metrics that are genuinely public/non-sensitive (e.g. an intentionally public status page).
|
||||
|
||||
## Impact
|
||||
|
||||
- Cloud account compromise (metadata creds via SSRF), internal network read access, and network mapping.
|
||||
- Theft of every backend credential Grafana/Prometheus/Alertmanager touches → lateral movement into DBs, Elasticsearch, cloud APIs.
|
||||
- RCE on the Grafana host (CVE-2024-9264) and arbitrary file read (CVE-2021-43798).
|
||||
- Kubernetes cluster recon → SA token / kubelet exposure → cluster compromise.
|
||||
- Alert suppression for detection evasion; secret/PII exposure via logs & traces.
|
||||
|
||||
## Pro Tips
|
||||
|
||||
1. Always fingerprint the version first (`/api/health`, `/api/v1/status/buildinfo`) — it decides RCE vs read vs recon.
|
||||
2. The exposed dashboard is never the finding; the pivot is. Chain to metadata creds, backend creds, or RCE before reporting.
|
||||
3. Prometheus `<secret>` masking is incomplete — hunt usernames and **URL-embedded creds** in `/api/v1/status/config` and `remote_write`.
|
||||
4. Grafana can query its own backends for you via the data source proxy — you don't need the plaintext password to exfil data.
|
||||
5. `*_build_info` and `kube_node_info` metrics hand you exact component versions — turn them straight into CVE targets.
|
||||
6. Pair with `ssrf`, `information_disclosure`, `kubernetes`, `aws`/`gcp`, and `authentication_jwt` skills; use `nuclei` templates (`grafana-*`, `prometheus-*`) for fast triage.
|
||||
7. On k8s, an exposed Prometheus/KSM often reveals the whole cluster topology and image versions with zero auth — prioritize it as a recon multiplier.
|
||||
|
||||
## Summary
|
||||
|
||||
Grafana and Prometheus are pivot engines, not endpoints. Grafana holds plaintext-recoverable credentials for every backend, proxies arbitrary server-side requests by default (SSRF → cloud metadata), reads arbitrary files (CVE-2021-43798), and can hit RCE (CVE-2024-9264). Prometheus/Alertmanager expose internal inventory, versions, and scrape/receiver credentials with no auth. Treat any reachable observability service as a launch point into the internal network, cloud account, databases, and cluster — and prove the pivot.
|
||||
@@ -365,6 +365,23 @@ agent-browser dialog accept "text" # accept with prompt input
|
||||
agent-browser dialog dismiss # cancel
|
||||
```
|
||||
|
||||
## Readiness & recovery
|
||||
|
||||
The first `agent-browser open` in a session launches the headless-Chrome
|
||||
daemon; later commands reuse it. Distinguish the two failure modes and react
|
||||
differently — do **not** blindly re-run the same failing command in a loop:
|
||||
|
||||
- **Daemon / connection failure** (`Failed to connect`, `connection refused`,
|
||||
socket missing, `browser not running`): the daemon isn't up or has died. Run
|
||||
`agent-browser doctor` (add `--fix` if it reports repairable problems), then
|
||||
re-open the page. Retrying the original command unchanged will keep failing.
|
||||
- **Malformed command** (`Unknown command`, `Ref not found`, bad flag): fix the
|
||||
command itself — re-snapshot for fresh refs, or correct the syntax.
|
||||
|
||||
Invoke `agent-browser` directly through `exec_command`; there is no need to wrap
|
||||
it in an extra `sh -c "..."` / `bash -lc "..."` layer, which only adds shell
|
||||
quoting and startup-file pitfalls.
|
||||
|
||||
## Diagnosing install issues
|
||||
|
||||
If a command fails unexpectedly (`Unknown command`, `Failed to connect`,
|
||||
|
||||
@@ -92,10 +92,18 @@ For iterative exploit work, put code in a file:
|
||||
|
||||
## Installing extra packages
|
||||
|
||||
The sandbox's Python lives in `/app/.venv`. To add a one-off dependency
|
||||
for an exploit script, use `uv` (already in the image and much faster
|
||||
than pip):
|
||||
The sandbox's Python lives in `/app/.venv`, and it is the active virtualenv
|
||||
(`python3` / `pip` already resolve to it). The following common libraries are
|
||||
**pre-installed** — import them directly, no install step needed:
|
||||
`requests`, `httpx`, `beautifulsoup4` (`bs4`), `lxml`, `pyjwt` (`jwt`),
|
||||
`cryptography`.
|
||||
|
||||
To add a one-off dependency for an exploit script, use `uv` (already in the
|
||||
image and much faster than pip):
|
||||
|
||||
```bash
|
||||
uv pip install --python /app/.venv/bin/python <package>
|
||||
```
|
||||
|
||||
Plain `pip install <package>` also works because the venv is active. Install
|
||||
before you import, so scripts don't fail with `ModuleNotFoundError`.
|
||||
|
||||
+100
-39
@@ -21,6 +21,8 @@ from caido_sdk_client.types import (
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from caido_sdk_client import Client as CaidoClient
|
||||
|
||||
|
||||
@@ -42,6 +44,7 @@ _SITEMAP_PAGE_SIZE = 30
|
||||
|
||||
_DEFAULT_CAIDO_URL = "http://127.0.0.1:48080"
|
||||
_CLIENT_CACHE: dict[str, Client] = {}
|
||||
_CLIENT_LOCK = asyncio.Lock()
|
||||
_REQ_FIELD_MAP: dict[SortBy, tuple[str, str]] = {
|
||||
"timestamp": ("req", "created_at"),
|
||||
"host": ("req", "host"),
|
||||
@@ -81,19 +84,46 @@ def _login_as_guest() -> str:
|
||||
return str(payload["data"]["loginAsGuest"]["token"]["accessToken"])
|
||||
|
||||
|
||||
async def get_client() -> Client:
|
||||
if client := _CLIENT_CACHE.get("default"):
|
||||
return client
|
||||
|
||||
async def _new_client() -> Client:
|
||||
token = await asyncio.to_thread(_login_as_guest)
|
||||
client = Client(caido_url(), auth=TokenAuthOptions(token=token))
|
||||
await client.connect()
|
||||
_CLIENT_CACHE["default"] = client
|
||||
return client
|
||||
|
||||
|
||||
async def get_client() -> Client:
|
||||
"""Return the shared Caido client, creating it under a lock if needed.
|
||||
|
||||
The lock prevents two concurrent callers from each building a client and
|
||||
racing ``connect()`` on the same transport ("Transport is already
|
||||
connected").
|
||||
"""
|
||||
async with _CLIENT_LOCK:
|
||||
client = _CLIENT_CACHE.get("default")
|
||||
if client is None:
|
||||
client = await _new_client()
|
||||
_CLIENT_CACHE["default"] = client
|
||||
return client
|
||||
|
||||
|
||||
async def call_with_client[T](fn: Callable[[Client], Awaitable[T]]) -> T:
|
||||
"""Run ``fn`` against the shared client, serialized through ``_CLIENT_LOCK``.
|
||||
|
||||
The Caido GraphQL transport is not safe for concurrent use: two in-flight
|
||||
requests race and raise "Transport is already connected". Serializing every
|
||||
proxy call through the lock prevents that.
|
||||
"""
|
||||
async with _CLIENT_LOCK:
|
||||
client = _CLIENT_CACHE.get("default")
|
||||
if client is None:
|
||||
client = await _new_client()
|
||||
_CLIENT_CACHE["default"] = client
|
||||
return await fn(client)
|
||||
|
||||
|
||||
async def close_client() -> None:
|
||||
client = _CLIENT_CACHE.pop("default", None)
|
||||
async with _CLIENT_LOCK:
|
||||
client = _CLIENT_CACHE.pop("default", None)
|
||||
if client is None:
|
||||
return
|
||||
await client.aclose()
|
||||
@@ -385,19 +415,23 @@ async def list_requests(
|
||||
sort_order: SortOrder = "desc",
|
||||
scope_id: str | None = None,
|
||||
) -> Any:
|
||||
return await list_requests_with_client(
|
||||
await get_client(),
|
||||
httpql_filter=httpql_filter,
|
||||
first=first,
|
||||
after=after,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
scope_id=scope_id,
|
||||
return await call_with_client(
|
||||
lambda client: list_requests_with_client(
|
||||
client,
|
||||
httpql_filter=httpql_filter,
|
||||
first=first,
|
||||
after=after,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
scope_id=scope_id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def view_request(request_id: str, *, part: RequestPart = "request") -> Any:
|
||||
return await get_request_with_client(await get_client(), request_id, part=part)
|
||||
return await call_with_client(
|
||||
lambda client: get_request_with_client(client, request_id, part=part)
|
||||
)
|
||||
|
||||
|
||||
async def repeat_request(
|
||||
@@ -406,22 +440,26 @@ async def repeat_request(
|
||||
modifications: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
mods = modifications or {}
|
||||
result = await get_request_with_client(await get_client(), request_id, part="request")
|
||||
if result is None or result.request.raw is None:
|
||||
raise ValueError(f"Request {request_id} not found")
|
||||
|
||||
original = result.request
|
||||
raw_str = result.request.raw.decode("utf-8", errors="replace")
|
||||
components = parse_raw_request(raw_str)
|
||||
full_url = full_url_from_components(original, components, mods)
|
||||
modified = apply_modifications(components, mods, full_url)
|
||||
connection, raw = build_raw_request(
|
||||
method=modified["method"],
|
||||
url=modified["url"],
|
||||
headers=modified["headers"],
|
||||
body=modified["body"],
|
||||
)
|
||||
return await replay_send_raw(await get_client(), raw=raw, connection=connection)
|
||||
async def _run(client: CaidoClient) -> dict[str, Any]:
|
||||
result = await get_request_with_client(client, request_id, part="request")
|
||||
if result is None or result.request.raw is None:
|
||||
raise ValueError(f"Request {request_id} not found")
|
||||
|
||||
original = result.request
|
||||
raw_str = result.request.raw.decode("utf-8", errors="replace")
|
||||
components = parse_raw_request(raw_str)
|
||||
full_url = full_url_from_components(original, components, mods)
|
||||
modified = apply_modifications(components, mods, full_url)
|
||||
connection, raw = build_raw_request(
|
||||
method=modified["method"],
|
||||
url=modified["url"],
|
||||
headers=modified["headers"],
|
||||
body=modified["body"],
|
||||
)
|
||||
return await replay_send_raw(client, raw=raw, connection=connection)
|
||||
|
||||
return await call_with_client(_run)
|
||||
|
||||
|
||||
async def scope_rules(
|
||||
@@ -432,7 +470,28 @@ async def scope_rules(
|
||||
scope_id: str | None = None,
|
||||
scope_name: str | None = None,
|
||||
) -> Any:
|
||||
client = await get_client()
|
||||
async def _run(client: CaidoClient) -> Any:
|
||||
return await _scope_rules_with_client(
|
||||
client,
|
||||
action,
|
||||
allowlist=allowlist,
|
||||
denylist=denylist,
|
||||
scope_id=scope_id,
|
||||
scope_name=scope_name,
|
||||
)
|
||||
|
||||
return await call_with_client(_run)
|
||||
|
||||
|
||||
async def _scope_rules_with_client(
|
||||
client: CaidoClient,
|
||||
action: ScopeAction,
|
||||
*,
|
||||
allowlist: list[str] | None = None,
|
||||
denylist: list[str] | None = None,
|
||||
scope_id: str | None = None,
|
||||
scope_name: str | None = None,
|
||||
) -> Any:
|
||||
if action == "list":
|
||||
result = await scope_list(client)
|
||||
elif action == "get":
|
||||
@@ -651,18 +710,20 @@ async def list_sitemap(
|
||||
page: int = 1,
|
||||
page_size: int = _SITEMAP_PAGE_SIZE,
|
||||
) -> dict[str, Any]:
|
||||
return await list_sitemap_with_client(
|
||||
await get_client(),
|
||||
scope_id=scope_id,
|
||||
parent_id=parent_id,
|
||||
depth=depth,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
return await call_with_client(
|
||||
lambda client: list_sitemap_with_client(
|
||||
client,
|
||||
scope_id=scope_id,
|
||||
parent_id=parent_id,
|
||||
depth=depth,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def view_sitemap_entry(entry_id: str) -> dict[str, Any]:
|
||||
return await view_sitemap_entry_with_client(await get_client(), entry_id)
|
||||
return await call_with_client(lambda client: view_sitemap_entry_with_client(client, entry_id))
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
+74
-32
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
@@ -19,6 +20,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from caido_sdk_client import Client
|
||||
|
||||
from strix.tools.proxy.caido_api import (
|
||||
@@ -38,12 +41,23 @@ else:
|
||||
|
||||
ScopeAction = Literal["get", "list", "create", "update", "delete"]
|
||||
|
||||
# All agents in a scan share one host-side Caido client whose GraphQL transport
|
||||
# is not concurrency-safe (parallel calls raise "Transport is already
|
||||
# connected"). Serialize every host-side proxy call through this lock.
|
||||
_CAIDO_CALL_LOCK = asyncio.Lock()
|
||||
|
||||
|
||||
def _ctx_client(ctx: RunContextWrapper) -> Client | None:
|
||||
inner = ctx.context if isinstance(ctx.context, dict) else {}
|
||||
return inner.get("caido_client")
|
||||
|
||||
|
||||
async def _call[T](client: Client, fn: Callable[[Client], Awaitable[T]]) -> T:
|
||||
"""Run ``fn`` against the shared client, serialized under ``_CAIDO_CALL_LOCK``."""
|
||||
async with _CAIDO_CALL_LOCK:
|
||||
return await fn(client)
|
||||
|
||||
|
||||
def _to_tool_json(value: Any) -> Any:
|
||||
"""Recursively convert SDK dataclasses/Pydantic objects to tool JSON values."""
|
||||
if value is None or isinstance(value, str | int | float | bool):
|
||||
@@ -146,14 +160,17 @@ async def list_requests(
|
||||
return _no_client()
|
||||
|
||||
try:
|
||||
connection = await caido_api.list_requests_with_client(
|
||||
connection = await _call(
|
||||
client,
|
||||
httpql_filter=httpql_filter,
|
||||
first=first,
|
||||
after=after,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
scope_id=scope_id,
|
||||
lambda client: caido_api.list_requests_with_client(
|
||||
client,
|
||||
httpql_filter=httpql_filter,
|
||||
first=first,
|
||||
after=after,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
scope_id=scope_id,
|
||||
),
|
||||
)
|
||||
|
||||
entries = []
|
||||
@@ -249,7 +266,10 @@ async def view_request(
|
||||
return _no_client()
|
||||
|
||||
try:
|
||||
result = await caido_api.get_request_with_client(client, request_id, part=part)
|
||||
result = await _call(
|
||||
client,
|
||||
lambda client: caido_api.get_request_with_client(client, request_id, part=part),
|
||||
)
|
||||
if result is None:
|
||||
return json.dumps(
|
||||
{"success": False, "error": f"Request {request_id} not found"},
|
||||
@@ -364,15 +384,10 @@ async def repeat_request(
|
||||
return _no_client()
|
||||
mods = modifications or {}
|
||||
|
||||
try:
|
||||
async def _do(client: Client) -> dict[str, Any] | None:
|
||||
result = await caido_api.get_request_with_client(client, request_id, part="request")
|
||||
if result is None or result.request.raw is None:
|
||||
return json.dumps(
|
||||
{"success": False, "error": f"Request {request_id} not found"},
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
|
||||
return None
|
||||
original = result.request
|
||||
raw_str = result.request.raw.decode("utf-8", errors="replace")
|
||||
components = caido_api.parse_raw_request(raw_str)
|
||||
@@ -384,7 +399,16 @@ async def repeat_request(
|
||||
headers=modified["headers"],
|
||||
body=modified["body"],
|
||||
)
|
||||
replay = await caido_api.replay_send_raw(client, raw=raw, connection=connection)
|
||||
return await caido_api.replay_send_raw(client, raw=raw, connection=connection)
|
||||
|
||||
try:
|
||||
replay = await _call(client, _do)
|
||||
if replay is None:
|
||||
return json.dumps(
|
||||
{"success": False, "error": f"Request {request_id} not found"},
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
return _format_replay_tool_result(replay)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return _err("repeat_request", exc)
|
||||
@@ -441,12 +465,15 @@ async def list_sitemap(
|
||||
if client is None:
|
||||
return _no_client()
|
||||
try:
|
||||
payload = await caido_api.list_sitemap_with_client(
|
||||
payload = await _call(
|
||||
client,
|
||||
scope_id=scope_id,
|
||||
parent_id=parent_id,
|
||||
depth=depth,
|
||||
page=page,
|
||||
lambda client: caido_api.list_sitemap_with_client(
|
||||
client,
|
||||
scope_id=scope_id,
|
||||
parent_id=parent_id,
|
||||
depth=depth,
|
||||
page=page,
|
||||
),
|
||||
)
|
||||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
@@ -472,7 +499,10 @@ async def view_sitemap_entry(
|
||||
if client is None:
|
||||
return _no_client()
|
||||
try:
|
||||
payload = await caido_api.view_sitemap_entry_with_client(client, entry_id)
|
||||
payload = await _call(
|
||||
client,
|
||||
lambda client: caido_api.view_sitemap_entry_with_client(client, entry_id),
|
||||
)
|
||||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return _err("view_sitemap_entry", exc)
|
||||
@@ -530,7 +560,7 @@ async def scope_rules(
|
||||
|
||||
try:
|
||||
if action == "list":
|
||||
scopes = await caido_api.scope_list(client)
|
||||
scopes = await _call(client, caido_api.scope_list)
|
||||
return json.dumps(
|
||||
{"success": True, "scopes": [_to_tool_json(s) for s in scopes]},
|
||||
ensure_ascii=False,
|
||||
@@ -543,9 +573,11 @@ async def scope_rules(
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
scope = await caido_api.scope_get(client, scope_id)
|
||||
scope = await _call(client, lambda client: caido_api.scope_get(client, scope_id))
|
||||
return json.dumps(
|
||||
{"success": True, "scope": _to_tool_json(scope)}, ensure_ascii=False, default=str
|
||||
{"success": True, "scope": _to_tool_json(scope)},
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
if action == "create":
|
||||
if not scope_name:
|
||||
@@ -554,11 +586,16 @@ async def scope_rules(
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
scope = await caido_api.scope_create(
|
||||
client, name=scope_name, allowlist=allowlist, denylist=denylist
|
||||
scope = await _call(
|
||||
client,
|
||||
lambda client: caido_api.scope_create(
|
||||
client, name=scope_name, allowlist=allowlist, denylist=denylist
|
||||
),
|
||||
)
|
||||
return json.dumps(
|
||||
{"success": True, "scope": _to_tool_json(scope)}, ensure_ascii=False, default=str
|
||||
{"success": True, "scope": _to_tool_json(scope)},
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
if action == "update":
|
||||
if not scope_id or not scope_name:
|
||||
@@ -570,11 +607,16 @@ async def scope_rules(
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
scope = await caido_api.scope_update(
|
||||
client, scope_id, name=scope_name, allowlist=allowlist, denylist=denylist
|
||||
scope = await _call(
|
||||
client,
|
||||
lambda client: caido_api.scope_update(
|
||||
client, scope_id, name=scope_name, allowlist=allowlist, denylist=denylist
|
||||
),
|
||||
)
|
||||
return json.dumps(
|
||||
{"success": True, "scope": _to_tool_json(scope)}, ensure_ascii=False, default=str
|
||||
{"success": True, "scope": _to_tool_json(scope)},
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
if not scope_id:
|
||||
return json.dumps(
|
||||
@@ -582,7 +624,7 @@ async def scope_rules(
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
await caido_api.scope_delete(client, scope_id)
|
||||
await _call(client, lambda client: caido_api.scope_delete(client, scope_id))
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
|
||||
@@ -5,6 +5,23 @@ invocation the agent makes (nmap, ffuf, agent-browser, python3, …) goes
|
||||
through `exec_command`. `write_stdin` streams input to a still-running
|
||||
process started by an earlier `exec_command` (for interactive prompts).
|
||||
|
||||
## `write_stdin` requires a TTY-backed process
|
||||
|
||||
`exec_command` runs each command in a fresh **non-interactive** shell (plain
|
||||
pipes, no TTY) by default. `write_stdin` only works against a process that is
|
||||
still running **and** was started with a PTY. The canonical sequence is:
|
||||
|
||||
```text
|
||||
exec_command(cmd="python3", tty=true) # start a PTY-backed process
|
||||
write_stdin(session_id=<id>, chars="print(1)\n")
|
||||
```
|
||||
|
||||
Calling `write_stdin` on a command started with the default `tty=false`, or on
|
||||
a process that has already exited, fails with
|
||||
`stdin is not available for this process. Start the command with 'tty=true' in
|
||||
'exec_command' before using 'write_stdin'.` Use `tty=true` for REPLs,
|
||||
`ssh`/`nc`/`ftp`, `msfconsole`, or to deliver a Ctrl-C to a long-running job.
|
||||
|
||||
- **Implementation:** `agents.sandbox.capabilities.tools.shell_tool.ShellTool`
|
||||
(in the upstream `agents` SDK)
|
||||
- **Wired in:** `strix/agents/factory.py` — added per-run via the SDK
|
||||
|
||||
@@ -6,6 +6,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
|
||||
from strix.config.models import _configure_litellm_compatibility
|
||||
from strix.report.state import litellm_cost_callback
|
||||
@@ -42,3 +43,111 @@ def test_cost_callback_reads_usage_cost_from_mapping_response() -> None:
|
||||
litellm_cost_callback({}, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_called_once_with(0.125)
|
||||
|
||||
|
||||
def test_cost_callback_reads_byok_upstream_inference_cost() -> None:
|
||||
report_state = MagicMock()
|
||||
response = SimpleNamespace(
|
||||
usage=SimpleNamespace(
|
||||
cost=0,
|
||||
is_byok=True,
|
||||
cost_details=SimpleNamespace(upstream_inference_cost=6.75e-06),
|
||||
),
|
||||
_hidden_params={},
|
||||
)
|
||||
|
||||
with patch("strix.report.state.get_global_report_state", return_value=report_state):
|
||||
litellm_cost_callback({"response_cost": None}, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_called_once_with(6.75e-06)
|
||||
|
||||
|
||||
def test_cost_callback_sums_usage_cost_and_upstream_inference_cost() -> None:
|
||||
report_state = MagicMock()
|
||||
response = {
|
||||
"usage": {
|
||||
"cost": 0.01,
|
||||
"is_byok": True,
|
||||
"cost_details": {"upstream_inference_cost": 0.2},
|
||||
}
|
||||
}
|
||||
|
||||
with patch("strix.report.state.get_global_report_state", return_value=report_state):
|
||||
litellm_cost_callback({}, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_called_once_with(pytest.approx(0.21))
|
||||
|
||||
|
||||
def test_cost_callback_ignores_upstream_cost_for_non_byok_responses() -> None:
|
||||
report_state = MagicMock()
|
||||
response = {
|
||||
"usage": {
|
||||
"cost": 0.05,
|
||||
"is_byok": False,
|
||||
"cost_details": {"upstream_inference_cost": 0.04},
|
||||
}
|
||||
}
|
||||
|
||||
with patch("strix.report.state.get_global_report_state", return_value=report_state):
|
||||
litellm_cost_callback({}, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_called_once_with(0.05)
|
||||
|
||||
|
||||
def test_cost_callback_estimates_cost_with_provider_prefixed_model() -> None:
|
||||
report_state = MagicMock()
|
||||
response = {"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}
|
||||
kwargs = {
|
||||
"response_cost": None,
|
||||
"model": "anthropic/claude-sonnet-4.5",
|
||||
"litellm_params": {"custom_llm_provider": "openrouter"},
|
||||
}
|
||||
|
||||
def fake_completion_cost(**kwargs: object) -> float:
|
||||
if kwargs["model"] == "openrouter/anthropic/claude-sonnet-4.5":
|
||||
return 0.5
|
||||
raise ValueError(kwargs["model"])
|
||||
|
||||
with (
|
||||
patch("strix.report.state.get_global_report_state", return_value=report_state),
|
||||
patch("litellm.completion_cost", side_effect=fake_completion_cost),
|
||||
):
|
||||
litellm_cost_callback(kwargs, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_called_once_with(0.5)
|
||||
|
||||
|
||||
def test_cost_callback_estimates_cost_with_bare_model_fallback() -> None:
|
||||
report_state = MagicMock()
|
||||
response = {"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}
|
||||
kwargs = {
|
||||
"response_cost": None,
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"litellm_params": {"custom_llm_provider": "openrouter"},
|
||||
}
|
||||
|
||||
def fake_completion_cost(**kwargs: object) -> float:
|
||||
if kwargs["model"] == "gpt-4o-mini":
|
||||
return 0.025
|
||||
raise ValueError(kwargs["model"])
|
||||
|
||||
with (
|
||||
patch("strix.report.state.get_global_report_state", return_value=report_state),
|
||||
patch("litellm.completion_cost", side_effect=fake_completion_cost),
|
||||
):
|
||||
litellm_cost_callback(kwargs, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_called_once_with(0.025)
|
||||
|
||||
|
||||
def test_cost_callback_records_nothing_when_no_cost_available() -> None:
|
||||
report_state = MagicMock()
|
||||
response = {"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}
|
||||
|
||||
with (
|
||||
patch("strix.report.state.get_global_report_state", return_value=report_state),
|
||||
patch("litellm.completion_cost", side_effect=ValueError("unknown model")),
|
||||
):
|
||||
litellm_cost_callback({"response_cost": None, "model": "x/y"}, response)
|
||||
|
||||
report_state.record_observed_llm_cost.assert_not_called()
|
||||
|
||||
@@ -155,3 +155,33 @@ def test_make_model_settings_forces_required_for_anyllm_routed_openai_model() ->
|
||||
)
|
||||
|
||||
assert settings.tool_choice == "required"
|
||||
|
||||
|
||||
def test_make_model_settings_sets_request_timeout() -> None:
|
||||
settings = make_model_settings(
|
||||
"none",
|
||||
model_name="gpt-4o",
|
||||
request_timeout=300.0,
|
||||
)
|
||||
|
||||
assert settings.extra_args is not None
|
||||
assert settings.extra_args["timeout"] == 300.0
|
||||
|
||||
|
||||
def test_make_model_settings_omits_timeout_when_unset() -> None:
|
||||
settings = make_model_settings("none", model_name="gpt-4o")
|
||||
|
||||
assert settings.extra_args is None
|
||||
|
||||
|
||||
def test_make_model_settings_timeout_survives_reasoning_resolve() -> None:
|
||||
# Reasoning is resolved via ModelSettings.resolve(); the timeout in extra_args
|
||||
# must not be dropped when a reasoning override is merged in.
|
||||
settings = make_model_settings(
|
||||
"high",
|
||||
model_name="openai/o3",
|
||||
request_timeout=120.0,
|
||||
)
|
||||
|
||||
assert settings.extra_args is not None
|
||||
assert settings.extra_args["timeout"] == 120.0
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Tests for the model retry policy used by every agent model call.
|
||||
|
||||
The SDK's built-in ``http_status`` policy only retries errors that carry a known
|
||||
HTTP status code. Quota/billing (and other provider-side) failures often surface
|
||||
*inside* a streamed response as a bare error with no status code, so Strix adds a
|
||||
statusless retry policy to ``DEFAULT_MODEL_RETRY`` to keep them recoverable — the
|
||||
behavior the pre-SDK engine had.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from agents.retry import ModelRetryNormalizedError, RetryPolicyContext
|
||||
|
||||
from strix.config.models import DEFAULT_MODEL_RETRY, _retry_statusless_provider_errors
|
||||
|
||||
|
||||
def _context(normalized: ModelRetryNormalizedError) -> RetryPolicyContext:
|
||||
return RetryPolicyContext(
|
||||
error=RuntimeError("boom"),
|
||||
attempt=1,
|
||||
max_retries=5,
|
||||
stream=True,
|
||||
normalized=normalized,
|
||||
provider_advice=None,
|
||||
)
|
||||
|
||||
|
||||
def _retries(normalized: ModelRetryNormalizedError) -> bool:
|
||||
"""Evaluate the composed DEFAULT_MODEL_RETRY policy for a normalized error."""
|
||||
policy = DEFAULT_MODEL_RETRY.policy
|
||||
assert policy is not None
|
||||
decision = asyncio.run(policy(_context(normalized)))
|
||||
return bool(getattr(decision, "retry", decision))
|
||||
|
||||
|
||||
def test_statusless_error_is_retried() -> None:
|
||||
# A mid-stream quota/billing error arrives with no HTTP status code.
|
||||
assert _retries(ModelRetryNormalizedError(status_code=None)) is True
|
||||
|
||||
|
||||
def test_statusless_abort_is_not_retried() -> None:
|
||||
# A user/client cancellation must never be retried.
|
||||
assert _retries(ModelRetryNormalizedError(status_code=None, is_abort=True)) is False
|
||||
|
||||
|
||||
def test_client_error_is_not_retried() -> None:
|
||||
# A definitive 4xx client error (bad request/auth) is not recoverable.
|
||||
assert _retries(ModelRetryNormalizedError(status_code=400)) is False
|
||||
|
||||
|
||||
def test_rate_limit_and_server_errors_are_retried() -> None:
|
||||
for status in (429, 500, 502, 503, 504):
|
||||
assert _retries(ModelRetryNormalizedError(status_code=status)) is True
|
||||
|
||||
|
||||
def test_timeout_error_is_retried() -> None:
|
||||
# A stalled model stream trips the per-request read/inactivity timeout, which
|
||||
# the SDK normalizes as a timeout. DEFAULT_MODEL_RETRY must retry it so a hung
|
||||
# turn recovers instead of silently wedging the agent.
|
||||
assert _retries(ModelRetryNormalizedError(is_timeout=True)) is True
|
||||
assert _retries(ModelRetryNormalizedError(is_network_error=True)) is True
|
||||
|
||||
|
||||
def test_policy_helper_matches_statusless_only() -> None:
|
||||
assert _retry_statusless_provider_errors(_context(ModelRetryNormalizedError())) is True
|
||||
assert (
|
||||
_retry_statusless_provider_errors(_context(ModelRetryNormalizedError(status_code=400)))
|
||||
is False
|
||||
)
|
||||
assert (
|
||||
_retry_statusless_provider_errors(
|
||||
_context(ModelRetryNormalizedError(status_code=None, is_abort=True))
|
||||
)
|
||||
is False
|
||||
)
|
||||
+23
-1
@@ -3,8 +3,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from agents.model_settings import ModelSettings
|
||||
|
||||
from strix.config.models import RECOMMENDED_MODEL_NAMES, is_recommended_or_frontier_model
|
||||
from strix.config.models import (
|
||||
RECOMMENDED_MODEL_NAMES,
|
||||
is_recommended_or_frontier_model,
|
||||
request_timeout_extra_args,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", RECOMMENDED_MODEL_NAMES)
|
||||
@@ -12,6 +17,23 @@ def test_recommended_models_are_accepted(model_name: str) -> None:
|
||||
assert is_recommended_or_frontier_model(model_name)
|
||||
|
||||
|
||||
def test_request_timeout_extra_args_positive() -> None:
|
||||
assert request_timeout_extra_args(300) == {"timeout": 300}
|
||||
assert request_timeout_extra_args(10) == {"timeout": 10}
|
||||
|
||||
|
||||
def test_request_timeout_extra_args_survives_model_settings_json_dump() -> None:
|
||||
"""The Chat Completions and LiteLLM paths pydantic-serialize ModelSettings for
|
||||
their tracing span; a non-JSON-serializable timeout fails every turn there."""
|
||||
settings = ModelSettings(extra_args=request_timeout_extra_args(300))
|
||||
assert settings.to_json_dict()["extra_args"] == {"timeout": 300}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, 0, -1])
|
||||
def test_request_timeout_extra_args_disabled(value: float | None) -> None:
|
||||
assert request_timeout_extra_args(value) is None
|
||||
|
||||
|
||||
def test_recommended_models_are_matched_case_insensitively() -> None:
|
||||
assert is_recommended_or_frontier_model("Vertex_AI/Gemini-3-Pro-Preview")
|
||||
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Tests for the shared Caido client lifecycle and proxy call serialization.
|
||||
|
||||
Covers the caching + serialization guarantees of ``caido_api.call_with_client``
|
||||
(the sandbox-imported path) and ``proxy.tools._call`` (the host-side path). The
|
||||
Caido GraphQL transport is not concurrency-safe, so both paths must run one
|
||||
call at a time against the shared client.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from strix.tools.proxy import caido_api, tools
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, name: str) -> None:
|
||||
self.name = name
|
||||
self.closed = False
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_cache() -> Iterator[None]:
|
||||
caido_api._CLIENT_CACHE.clear()
|
||||
yield
|
||||
caido_api._CLIENT_CACHE.clear()
|
||||
|
||||
|
||||
async def test_call_with_client_reuses_cached_client(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
cached = _FakeClient("cached")
|
||||
caido_api._CLIENT_CACHE["default"] = cast("Any", cached)
|
||||
|
||||
async def _new() -> Any:
|
||||
raise AssertionError("_new_client must not run when a client is cached")
|
||||
|
||||
monkeypatch.setattr(caido_api, "_new_client", _new)
|
||||
|
||||
seen: dict[str, Any] = {}
|
||||
|
||||
async def fn(client: Any) -> str:
|
||||
seen["client"] = client
|
||||
return "ok"
|
||||
|
||||
assert await caido_api.call_with_client(fn) == "ok"
|
||||
assert seen["client"] is cached
|
||||
|
||||
|
||||
async def test_call_with_client_creates_and_caches_when_empty(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
created = _FakeClient("fresh")
|
||||
|
||||
async def _new() -> Any:
|
||||
return created
|
||||
|
||||
monkeypatch.setattr(caido_api, "_new_client", _new)
|
||||
|
||||
seen: dict[str, Any] = {}
|
||||
|
||||
async def fn(client: Any) -> str:
|
||||
seen["client"] = client
|
||||
return "ok"
|
||||
|
||||
assert await caido_api.call_with_client(fn) == "ok"
|
||||
assert seen["client"] is created
|
||||
assert caido_api._CLIENT_CACHE["default"] is created
|
||||
|
||||
|
||||
async def test_failed_init_does_not_poison_cache(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
async def _new() -> Any:
|
||||
raise ConnectionRefusedError("caido not up yet")
|
||||
|
||||
monkeypatch.setattr(caido_api, "_new_client", _new)
|
||||
|
||||
async def fn(_client: Any) -> str:
|
||||
return "unreachable"
|
||||
|
||||
with pytest.raises(ConnectionRefusedError):
|
||||
await caido_api.call_with_client(fn)
|
||||
assert "default" not in caido_api._CLIENT_CACHE
|
||||
|
||||
|
||||
async def test_call_with_client_propagates_errors() -> None:
|
||||
cached = _FakeClient("cached")
|
||||
caido_api._CLIENT_CACHE["default"] = cast("Any", cached)
|
||||
|
||||
async def fn(_client: Any) -> str:
|
||||
raise ValueError("Invalid HTTPQL filter")
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid HTTPQL"):
|
||||
await caido_api.call_with_client(fn)
|
||||
assert caido_api._CLIENT_CACHE["default"] is cached
|
||||
|
||||
|
||||
async def test_call_with_client_serializes_concurrent_calls(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
caido_api._CLIENT_CACHE["default"] = cast("Any", _FakeClient("shared"))
|
||||
|
||||
async def _new() -> Any:
|
||||
raise AssertionError("no new client expected")
|
||||
|
||||
monkeypatch.setattr(caido_api, "_new_client", _new)
|
||||
|
||||
state = {"active": 0, "max": 0}
|
||||
|
||||
async def fn(_client: Any) -> str:
|
||||
state["active"] += 1
|
||||
state["max"] = max(state["max"], state["active"])
|
||||
await asyncio.sleep(0.01)
|
||||
state["active"] -= 1
|
||||
return "ok"
|
||||
|
||||
await asyncio.gather(*(caido_api.call_with_client(fn) for _ in range(6)))
|
||||
assert state["max"] == 1
|
||||
|
||||
|
||||
async def test_host_call_serializes_concurrent_calls() -> None:
|
||||
client = _FakeClient("host")
|
||||
state = {"active": 0, "max": 0}
|
||||
|
||||
async def fn(_client: Any) -> str:
|
||||
state["active"] += 1
|
||||
state["max"] = max(state["max"], state["active"])
|
||||
await asyncio.sleep(0.01)
|
||||
state["active"] -= 1
|
||||
return "ok"
|
||||
|
||||
await asyncio.gather(*(tools._call(cast("Any", client), fn) for _ in range(6)))
|
||||
assert state["max"] == 1
|
||||
|
||||
|
||||
class _Ctx:
|
||||
def __init__(self, context: Any) -> None:
|
||||
self.context = context
|
||||
|
||||
|
||||
def test_ctx_client_returns_client_when_present() -> None:
|
||||
client = _FakeClient("host")
|
||||
got = tools._ctx_client(cast("Any", _Ctx({"caido_client": client})))
|
||||
assert got is client
|
||||
|
||||
|
||||
def test_ctx_client_returns_none_without_client() -> None:
|
||||
assert tools._ctx_client(cast("Any", _Ctx({}))) is None
|
||||
assert tools._ctx_client(cast("Any", _Ctx(None))) is None
|
||||
@@ -37,6 +37,7 @@ async def test_persistent_rate_limit_stops_gracefully(
|
||||
model="openai/gpt-4o",
|
||||
reasoning_effort="high",
|
||||
force_required_tool_choice=False,
|
||||
timeout=300,
|
||||
),
|
||||
runtime=types.SimpleNamespace(max_context_images=3),
|
||||
)
|
||||
|
||||
@@ -45,6 +45,7 @@ def _patch_engine_scaffold(
|
||||
model="openai/gpt-4o",
|
||||
reasoning_effort="high",
|
||||
force_required_tool_choice=False,
|
||||
timeout=300,
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(runner, "load_settings", lambda: settings)
|
||||
@@ -124,8 +125,7 @@ async def test_root_prompt_options_flow_into_root_agent(
|
||||
assert "https://example.com" in instructions_override
|
||||
assert "CUSTOM SCAN PROMPT" in instructions_override
|
||||
assert (
|
||||
"cannot expand, replace, or weaken authorized target constraints"
|
||||
in instructions_override
|
||||
"cannot expand, replace, or weaken authorized target constraints" in instructions_override
|
||||
)
|
||||
assert kwargs["system_prompt_context"] == {
|
||||
**scope_context,
|
||||
|
||||
Reference in New Issue
Block a user