diff --git a/strix/agents/prompts/system_prompt.jinja b/strix/agents/prompts/system_prompt.jinja index 97448a3b..0be45595 100644 --- a/strix/agents/prompts/system_prompt.jinja +++ b/strix/agents/prompts/system_prompt.jinja @@ -207,6 +207,9 @@ VALIDATION REQUIREMENTS: - Full validation required - no assumptions - Demonstrate concrete impact with evidence - Consider business context for severity assessment — check whether the target is a demo/sandbox environment or content meant to be public, and factor that in +- Score only the security impact demonstrated by the proof of concept. Reachability, missing authentication, scanner labels, and theoretical follow-on attacks do not by themselves justify non-None CVSS impact metrics +- Treat public metadata, internal-looking identifiers, source maps without secrets, and transport/configuration hygiene as observations unless validation proves unauthorized restricted-data access, modification, or service disruption +- Every non-None Confidentiality, Integrity, or Availability metric must map to explicit evidence in the report; use Scope Changed only for a demonstrated crossing of security authorities - Independent verification through subagent - Document complete attack chain - Keep going until you find something that matters diff --git a/strix/skills/vulnerabilities/information_disclosure.md b/strix/skills/vulnerabilities/information_disclosure.md index 583756b4..5c09f3ab 100644 --- a/strix/skills/vulnerabilities/information_disclosure.md +++ b/strix/skills/vulnerabilities/information_disclosure.md @@ -113,10 +113,13 @@ Information leaks accelerate exploitation by revealing code, configuration, iden ## Triage Rubric -- **Critical**: Credentials/keys; signed URL secrets; config dumps; unrestricted admin/observability panels -- **High**: Versions with reachable CVEs; cross-tenant data; caches serving cross-user content -- **Medium**: Internal paths/hosts enabling LFI/SSRF pivots; source maps revealing hidden endpoints -- **Low**: Generic headers, marketing versions, intended documentation without exploit path +- **Critical**: Direct disclosure of secrets that provide broad privileged control, with successful use validated where safe +- **High**: Direct disclosure of highly sensitive data, cross-tenant data, or credentials with serious demonstrated access +- **Medium**: Unauthorized access to a limited set of genuinely restricted data, or a fully validated chain from the disclosure to a concrete security consequence +- **Low**: Limited unauthorized disclosure with modest sensitivity and no serious direct consequence +- **Informational / no report**: Public or intended client data, generic headers, internal names or private addresses, versions without a reachable exploit, and source maps or diagnostics that expose no secrets or restricted source + +Do not assign Confidentiality Low merely because information helps reconnaissance. CVSS C:L requires actual access to restricted information. If a path, hostname, version, source map, schema, or debug value only suggests a possible second vulnerability, either validate that complete chain and score its demonstrated outcome or leave C:N and omit the vulnerability report. ## Exploitation Chains @@ -152,6 +155,7 @@ Information leaks accelerate exploitation by revealing code, configuration, iden 3. Attempt minimal, reversible exploitation or present a concrete step-by-step chain 4. Show reproducibility and minimal request set 5. Bound scope (user, tenant, environment) and data sensitivity classification +6. Map each non-None CVSS impact metric to evidence of actual restricted disclosure, modification, or service interruption ## False Positives diff --git a/strix/tools/reporting/tool.py b/strix/tools/reporting/tool.py index b7bb1885..598def57 100644 --- a/strix/tools/reporting/tool.py +++ b/strix/tools/reporting/tool.py @@ -126,23 +126,25 @@ def _validate_cwe(cwe: str) -> str | None: def _calculate_cvss(breakdown: dict[str, str]) -> tuple[float, str, str]: - try: - from cvss import CVSS3 + from cvss import CVSS3 - vector = ( - f"CVSS:3.1/AV:{breakdown['attack_vector']}/AC:{breakdown['attack_complexity']}/" - f"PR:{breakdown['privileges_required']}/UI:{breakdown['user_interaction']}/" - f"S:{breakdown['scope']}/C:{breakdown['confidentiality']}/" - f"I:{breakdown['integrity']}/A:{breakdown['availability']}" - ) - c = CVSS3(vector) - score = c.scores()[0] - severity = c.severities()[0].lower() - except Exception: - logger.exception("Failed to calculate CVSS") - return 7.5, "high", "" - else: - return score, severity, vector + vector = ( + f"CVSS:3.1/AV:{breakdown['attack_vector']}/AC:{breakdown['attack_complexity']}/" + f"PR:{breakdown['privileges_required']}/UI:{breakdown['user_interaction']}/" + f"S:{breakdown['scope']}/C:{breakdown['confidentiality']}/" + f"I:{breakdown['integrity']}/A:{breakdown['availability']}" + ) + + try: + cvss = CVSS3(vector) + score = cvss.scores()[0] + base_severity = cvss.severities()[0].lower() + except Exception as exc: + msg = f"Failed to calculate CVSS for validated vector: {vector}" + raise ValueError(msg) from exc + + severity = "info" if base_severity == "none" else base_severity + return score, severity, vector _REQUIRED_FIELDS = { @@ -233,7 +235,10 @@ async def _do_create( # noqa: PLR0912 if errors: return {"success": False, "error": "Validation failed", "errors": errors} - cvss_score, severity, _vector = _calculate_cvss(cvss_breakdown) + try: + cvss_score, severity, _vector = _calculate_cvss(cvss_breakdown) + except ValueError as exc: + return {"success": False, "error": "Validation failed", "errors": [str(exc)]} try: from strix.report.state import get_global_report_state @@ -377,6 +382,30 @@ async def create_vulnerability_report( lockfile/manifest that matches a published advisory. File those with ``create_dependency_report`` instead, never with this tool. + **Reporting and severity gate**: + + - A reachable endpoint, unusual response, weak configuration, or + reconnaissance artifact is not by itself a vulnerability. File a + report only when the PoC demonstrates an unauthorized security + consequence or a realistic, fully validated path to one. + - Score only the reasonable final impact supported by the PoC. Do + not score speculative pivots or consequences that require another + unverified vulnerability. + - Network reachability and missing authentication affect + exploitability; neither creates Confidentiality, Integrity, or + Availability impact by itself. + - Public metadata, internal-looking names or addresses, software + versions, intended client-side code, and source maps without + secrets or restricted source normally have ``C:N``. + - Configuration and transport observations require a realistic + attacker-controlled exploit and direct security impact. Client + errors, compatibility issues, fingerprinting, and attack-surface + discovery alone should not be filed as vulnerabilities. + - Before filing, verify that the impact narrative, PoC, and every + non-None CVSS impact metric describe the same demonstrated + consequence. When evidence is incomplete, lower the metric or + continue validation; never choose a higher value "to be safe." + Automatic LLM-based **deduplication** rejects reports that describe the same root cause on the same asset as an existing report. If you get a ``duplicate_of`` response, do NOT retry — move on to other @@ -430,6 +459,23 @@ async def create_vulnerability_report( - ``confidentiality`` / ``integrity`` / ``availability``: ``N`` / ``L`` / ``H`` + Derive the vector from the demonstrated attack, not the finding + category or a scanner/template severity: + + - ``C:L`` requires actual access to some restricted information. + Reconnaissance value alone is ``C:N``. ``C:H`` requires total + disclosure or limited disclosure with a direct serious impact, + such as a usable administrator credential or private key. + - ``I:L`` requires demonstrated unauthorized, limited modification; + ``I:H`` requires total or directly serious modification. Otherwise + use ``I:N``. + - ``A:L`` requires demonstrated performance degradation or service + interruption; ``A:H`` requires complete or directly serious + denial of the affected service. Otherwise use ``A:N``. + - Use ``S:C`` only when exploitation demonstrably crosses into a + component governed by a different security authority. A separate + backend, downstream effect, or third-party name is insufficient. + Example:: { @@ -502,7 +548,10 @@ async def create_vulnerability_report( (1-3 sentences) — it appears first in the report. Deep technical detail and root-cause analysis belong in ``technical_analysis``, not here. - impact: What an attacker achieves; business risk; data at risk. + impact: The unauthorized result demonstrated by the PoC, the + affected data or operation, and its scope. Keep plausible + but unverified follow-on risks separate; do not use them to + set CVSS metrics. target: Affected URL / domain / repository. technical_analysis: The mechanism and root cause. poc_description: Step-by-step reproduction (steps only, no code). diff --git a/tests/test_reporting_tool.py b/tests/test_reporting_tool.py new file mode 100644 index 00000000..55122334 --- /dev/null +++ b/tests/test_reporting_tool.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import pytest + +from strix.tools.reporting.tool import _calculate_cvss + + +def test_cvss_without_demonstrated_impact_is_informational() -> None: + score, severity, vector = _calculate_cvss( + { + "attack_vector": "N", + "attack_complexity": "L", + "privileges_required": "N", + "user_interaction": "N", + "scope": "U", + "confidentiality": "N", + "integrity": "N", + "availability": "N", + } + ) + + assert score == 0.0 + assert severity == "info" + assert vector == "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N" + + +def test_cvss_calculation_does_not_fabricate_high_severity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class BrokenCVSS: + def __init__(self, vector: str) -> None: + raise RuntimeError(vector) + + monkeypatch.setattr("cvss.CVSS3", BrokenCVSS) + + with pytest.raises(ValueError, match="Failed to calculate CVSS"): + _calculate_cvss( + { + "attack_vector": "N", + "attack_complexity": "L", + "privileges_required": "N", + "user_interaction": "N", + "scope": "U", + "confidentiality": "L", + "integrity": "N", + "availability": "N", + } + )