From 97336d53e45645789a9f733d5fa065d0229efa09 Mon Sep 17 00:00:00 2001 From: Alex Schapiro Date: Wed, 5 Aug 2026 17:39:06 +0000 Subject: [PATCH] feat(reporting): structured reachability evidence ladder for dependency CVE findings --- containers/Dockerfile | 4 +- .../tui/internal/render/dependency.go | 6 + .../skills/custom/dependency_cve_scanning.md | 76 +++++++++++- strix/tools/reporting/tool.py | 93 +++++++++++++- tests/test_reporting_fields.py | 116 ++++++++++++++++++ 5 files changed, 285 insertions(+), 10 deletions(-) diff --git a/containers/Dockerfile b/containers/Dockerfile index 4b3c84d2..9943266a 100644 --- a/containers/Dockerfile +++ b/containers/Dockerfile @@ -16,7 +16,8 @@ RUN mkdir -p /out/bin && \ go install -v github.com/projectdiscovery/katana/cmd/katana@latest && \ go install -v github.com/projectdiscovery/cvemap/cmd/vulnx@latest && \ go install -v github.com/jaeles-project/gospider@latest && \ - go install -v github.com/projectdiscovery/interactsh/cmd/interactsh-client@latest + go install -v github.com/projectdiscovery/interactsh/cmd/interactsh-client@latest && \ + go install -v golang.org/x/vuln/cmd/govulncheck@latest # --------------------------------------------------------------------------- # Runtime stage @@ -53,6 +54,7 @@ RUN apt-get update && \ nmap ncat ndiff \ sqlmap nuclei subfinder naabu ffuf \ nodejs npm pipx \ + golang-go \ libcap2-bin \ gdb \ libnss3-tools \ diff --git a/strix/interface/tui/internal/render/dependency.go b/strix/interface/tui/internal/render/dependency.go index 78bc6cd4..f41bddc3 100644 --- a/strix/interface/tui/internal/render/dependency.go +++ b/strix/interface/tui/internal/render/dependency.go @@ -57,6 +57,12 @@ func renderDependencyReport(args map[string]any, result any) string { section("Description", StringValue(args["description"])) section("Impact", StringValue(args["impact"])) section("Technical Analysis", StringValue(args["technical_analysis"])) + if reach := StringValue(args["reachability"]); reach != "" && reach != "unknown" { + b.WriteString("\n\n" + Bold(Field).Render("Usage evidence: ") + reach) + if ev := StringValue(args["reachability_evidence"]); ev != "" { + b.WriteString("\n" + ev) + } + } section("Assumptions", StringValue(args["assumptions"])) section("Remediation", StringValue(args["remediation_steps"])) if title == "" { diff --git a/strix/skills/custom/dependency_cve_scanning.md b/strix/skills/custom/dependency_cve_scanning.md index 448af0e8..59e37ecc 100644 --- a/strix/skills/custom/dependency_cve_scanning.md +++ b/strix/skills/custom/dependency_cve_scanning.md @@ -110,15 +110,76 @@ resolution (npm `overrides` / yarn `resolutions` / pnpm `pnpm.overrides` / Maven `dependencyManagement` / Gradle resolution strategy / `go mod edit`), not just "upgrade to ". +### Usage / reachability analysis (required for every dependency CVE) + +For every CVE you are about to report, run a static usage analysis and record +the result in the structured `reachability` + `reachability_evidence` fields. +The level is an **evidence ladder, never an exploitability verdict** — claim +only what you proved, and cite the proof. It never changes severity (that is +`advisory_cvss` alone); it exists so the reader can prioritize. + +**Go — use govulncheck (real call-graph analysis):** + +```bash +# Symbol-level: reports only vulnerabilities whose vulnerable functions are +# actually reachable from application code. Needs the Go toolchain + module +# deps; if either is missing, fall back to the checks below rather than +# claiming a level. +if command -v govulncheck >/dev/null && go version >/dev/null 2>&1; then + govulncheck -format json ./... > "$ART/govulncheck.json" || true +fi +``` + +- A finding with a call stack ⇒ `reachability=reachable_call_path`, put the + call-path excerpt (entrypoint → vulnerable function) in + `reachability_evidence`. +- Listed as affecting a required module but with no reachable symbol ⇒ fall + back to the import/symbol checks below (`imported` / `not_imported`). + +**All other ecosystems — import check, then symbol match:** + +1. **Import check.** Search application code (exclude lockfiles, vendored + deps, `node_modules`, build output) for imports of the vulnerable package: + `ast-grep`/`rg` for `import`/`require`/`from X import` of the package (and + its ecosystem import name, which may differ from the registry name, e.g. + `PyYAML` → `yaml`). No hits ⇒ `not_imported`, with the search scope stated + in `reachability_evidence`. For a **transitive** dependency, the check is + whether application code imports it directly; if not, it is reachable only + through the direct dependency — check whether the direct dep's usage can + hit it (if unclear, use `imported` when the direct dep is used at all). +2. **Symbol match.** Read the advisory (GHSA/NVD/OSV `affected[].ecosystem_specific.imports` or the + advisory text) for the affected functions/classes/APIs. Search application + code for those symbols (`ast-grep` pattern or `rg -n`). Hits ⇒ + `vulnerable_symbol_used`, with repo-relative `file:line` of each hit (up + to a handful) in `reachability_evidence`. Imported but no affected-symbol + usage found (or the advisory names no symbols) ⇒ `imported`. +3. If the analysis was not performed or is inconclusive (obfuscated code, + dynamic loading, unparsable sources) ⇒ `unknown` and say why in + `assumptions`. + +Cheap-first budgeting: the import check is one search per package — always do +it. Do the symbol match at least for every `critical`/`high`/KEV CVE; batch +the searches. Never let this analysis stall reporting — `unknown` with a +reason beats an unverified claim. + +Anti-overclaim rules: + +- `not_imported` still does NOT mean safe (dynamic `import()`/reflection/ + framework wiring evade static search) — never phrase it as "not exploitable". +- `reachable_call_path` is reserved for call-graph tools (govulncheck); a + symbol grep hit is `vulnerable_symbol_used`, no matter how convinced you are. +- The tool rejects any level other than `unknown` without + `reachability_evidence`. + ### Reachability is a confidence modifier, not a gate Do NOT suppress or downgrade a known CVE just because you could not prove the vulnerable code path is reachable. Report it, set `advisory_cvss` from the -advisory, and use `assumptions` to note reachability (e.g. "the vulnerable -`template()` API does not appear to be imported in application code, so practical -exploitability is uncertain"). If you *can* show reachability or chain it into a -dynamic exploit, do that and report it as a normal dynamic finding with -`create_vulnerability_report` instead. +advisory, record the usage analysis in `reachability`/`reachability_evidence`, +and use `assumptions` for anything softer. If you *can* actually trigger the +vulnerable path or chain it into a dynamic exploit, additionally report that +as a normal dynamic finding with `create_vulnerability_report` (the standalone +CVE stays in its own `create_dependency_report`). ## Reporting @@ -152,7 +213,8 @@ findings and rejects empty PoC fields): - Set `cwe` to the most specific `CWE-NNN` when the advisory names one. - Do NOT cap severity at LOW just because there is no dynamic reproduction — use the advisory score. -- Use `assumptions` for reachability/exploitability caveats. +- Set `reachability` + `reachability_evidence` from the usage analysis above; + use `assumptions` for anything softer (confidence, caveats, analysis limits). Verify the CVE with `web_search` when available before reporting. Never guess or hallucinate a CVE id. @@ -168,3 +230,5 @@ hallucinate a CVE id. - Do not silently drop a known CVE because it lacks a dynamic PoC — that is the exact failure this skill prevents. - Do not downgrade advisory severity for lack of dynamic reproduction. +- Do not claim a `reachability` level the evidence does not prove — `unknown` + with a reason is always acceptable; an overclaimed level never is. diff --git a/strix/tools/reporting/tool.py b/strix/tools/reporting/tool.py index a1425315..e2c08c1e 100644 --- a/strix/tools/reporting/tool.py +++ b/strix/tools/reporting/tool.py @@ -719,6 +719,17 @@ def _dependency_severity(advisory_cvss: float | None) -> tuple[float, str]: return score, "none" +_VALID_REACHABILITY = frozenset( + { + "not_imported", + "imported", + "vulnerable_symbol_used", + "reachable_call_path", + "unknown", + } +) + + def _build_dependency_metadata( *, package_name: str, @@ -727,6 +738,8 @@ def _build_dependency_metadata( fixed_version: str | None, introduced_by: str | None, dependency_path: str | None, + reachability: str | None = None, + reachability_evidence: str | None = None, ) -> dict[str, str]: metadata = { "package_name": package_name.strip(), @@ -740,9 +753,25 @@ def _build_dependency_metadata( metadata["introduced_by"] = introduced_by.strip() if dependency_path and dependency_path.strip(): metadata["dependency_path"] = dependency_path.strip() + # "unknown" is the absent case — omitting it keeps the jsonb contract clean, + # and evidence without a level would have nothing to qualify. + if reachability and reachability.strip() and reachability.strip() != "unknown": + metadata["reachability"] = reachability.strip() + if reachability_evidence and reachability_evidence.strip(): + metadata["reachability_evidence"] = reachability_evidence.strip() return metadata +_REACHABILITY_EVIDENCE_LABELS = { + "not_imported": "not imported by application code", + "imported": "imported by application code; affected API usage unconfirmed", + "vulnerable_symbol_used": "the advisory's affected API is used in application code", + "reachable_call_path": ( + "a call path from application code to the vulnerable function was proven" + ), +} + + def _build_dependency_evidence( *, cve: str, @@ -751,6 +780,8 @@ def _build_dependency_evidence( fixed_version: str | None, introduced_by: str | None, dependency_path: str | None, + reachability: str | None = None, + reachability_evidence: str | None = None, ) -> str: evidence = ( f"**Advisory evidence:** `{cve}` applies to `{package_name}` " @@ -765,6 +796,15 @@ def _build_dependency_evidence( ) if dependency_path and dependency_path.strip(): evidence += f"\n\n**Dependency chain:** `{dependency_path.strip()}`" + label = _REACHABILITY_EVIDENCE_LABELS.get((reachability or "").strip().lower()) + if label: + evidence += f"\n\n**Usage analysis:** {label}." + if reachability_evidence and reachability_evidence.strip(): + evidence += f" {reachability_evidence.strip()}" + evidence += ( + " This is a prioritization signal from static analysis, not a" + " proof of exploitability or of safety." + ) return evidence @@ -787,6 +827,8 @@ async def _do_create_dependency( # noqa: PLR0912 fix_effort: str, introduced_by: str | None = None, dependency_path: str | None = None, + reachability: str = "unknown", + reachability_evidence: str | None = None, agent_id: str | None = None, agent_name: str | None = None, ) -> dict[str, Any]: @@ -823,6 +865,18 @@ async def _do_create_dependency( # noqa: PLR0912 f"Invalid fix_effort: {fix_effort!r}. Must be one of: {sorted(_VALID_FIX_EFFORT)}" ) + reachability = (reachability or "unknown").strip().lower() + if reachability not in _VALID_REACHABILITY: + errors.append( + f"Invalid reachability: {reachability!r}. Must be one of: {sorted(_VALID_REACHABILITY)}" + ) + elif reachability != "unknown" and not (reachability_evidence or "").strip(): + errors.append( + "reachability_evidence is required when reachability is not 'unknown': " + "cite the concrete proof (import file:line, matched symbol usage, or " + "govulncheck call path). Never claim a reachability level without evidence." + ) + if advisory_cvss is None: errors.append( "advisory_cvss is required: read the published advisory base score " @@ -843,6 +897,8 @@ async def _do_create_dependency( # noqa: PLR0912 fixed_version=fixed_version, introduced_by=introduced_by, dependency_path=dependency_path, + reachability=reachability, + reachability_evidence=reachability_evidence, ) evidence = _build_dependency_evidence( cve=parsed_cve, @@ -851,6 +907,8 @@ async def _do_create_dependency( # noqa: PLR0912 fixed_version=fixed_version, introduced_by=introduced_by, dependency_path=dependency_path, + reachability=reachability, + reachability_evidence=reachability_evidence, ) try: @@ -949,6 +1007,8 @@ async def create_dependency_report( fix_effort: str = "low", introduced_by: str | None = None, dependency_path: str | None = None, + reachability: str = "unknown", + reachability_evidence: str | None = None, ) -> str: """File a known-CVE dependency (SCA) finding — one report per CVE x package. @@ -973,9 +1033,26 @@ async def create_dependency_report( - Re-reporting the same CVE/package already filed. **Reachability**: do NOT silently downgrade or suppress a finding - because the vulnerable code path may be unreachable — instead state - reachability as an ``assumptions`` / confidence factor. Report the - finding; let the reader weigh exploitability. + because the vulnerable code path may be unreachable — report it, and + record what the usage analysis showed via the structured + ``reachability`` + ``reachability_evidence`` fields (see the + dependency-cve-scanning skill for the analysis procedure). The level + is an evidence ladder, never an exploitability verdict: + + - ``not_imported`` — the package is never imported/required by + application code (strongest de-prioritization signal; still not + proof of safety — dynamic loading, reflection, or framework wiring + can evade static search). + - ``imported`` — application code imports the package, but usage of + the advisory's affected API was not confirmed. + - ``vulnerable_symbol_used`` — the advisory's affected + function/class/API appears in application code. + - ``reachable_call_path`` — a call-graph tool (e.g. ``govulncheck``) + proved a path from application code to the vulnerable function. + - ``unknown`` — usage analysis was not performed or was inconclusive. + + Severity is still derived solely from ``advisory_cvss`` — the + reachability level never changes the rating, only prioritization. **Formatting**: use markdown in text fields (``**bold**``, ``inline code`` for package/version identifiers, fenced code blocks for @@ -1010,6 +1087,14 @@ async def create_dependency_report( to the vulnerable package, joined with `` > `` (e.g. ``express@4.18.1 > body-parser@1.20.0 > qs@6.10.2``). Omit for direct dependencies. + reachability: Usage-evidence level from static analysis — one of + ``not_imported`` / ``imported`` / ``vulnerable_symbol_used`` / + ``reachable_call_path`` / ``unknown``. Claim only what the + evidence proves; when in doubt use ``unknown``. + reachability_evidence: The concrete proof for the claimed level + (required for any level other than ``unknown``): repo-relative + ``file:line`` of the import or symbol usage, the matched + advisory symbols, or the govulncheck call-path excerpt. """ agent_id, agent_name = _caller_identity(ctx) @@ -1031,6 +1116,8 @@ async def create_dependency_report( fix_effort=fix_effort, introduced_by=introduced_by, dependency_path=dependency_path, + reachability=reachability, + reachability_evidence=reachability_evidence, agent_id=agent_id, agent_name=agent_name, ) diff --git a/tests/test_reporting_fields.py b/tests/test_reporting_fields.py index 7bef970b..027c0a9f 100644 --- a/tests/test_reporting_fields.py +++ b/tests/test_reporting_fields.py @@ -255,6 +255,120 @@ async def test_dependency_report_with_zero_cvss_remains_low_severity( assert report["cvss"] == 0.0 +async def test_dependency_report_records_reachability(report_state: ReportState) -> None: + result = await _do_create_dependency( + title="CVE-2021-23337 in lodash 4.17.20", + description="Command injection via template.", + target="repo/package.json", + cve="CVE-2021-23337", + package_name="lodash", + installed_version="4.17.20", + impact="Command injection where template is used.", + remediation_steps="Upgrade to 4.17.21.", + assumptions="Assumes the template sink is reachable.", + package_ecosystem="npm", + fixed_version="4.17.21", + cwe=None, + advisory_cvss=7.2, + technical_analysis=None, + fix_effort="low", + reachability="vulnerable_symbol_used", + reachability_evidence="src/render.ts:14 calls `_.template()`.", + ) + + assert result["success"] is True + report = report_state.vulnerability_reports[0] + assert report["dependency_metadata"]["reachability"] == "vulnerable_symbol_used" + assert ( + report["dependency_metadata"]["reachability_evidence"] + == "src/render.ts:14 calls `_.template()`." + ) + assert "**Usage analysis:**" in report["evidence"] + assert "not a proof of exploitability or of safety" in report["evidence"] + # The level must never influence the rating — that stays advisory_cvss only. + assert report["severity"] == "high" + + +async def test_dependency_report_rejects_reachability_without_evidence( + report_state: ReportState, +) -> None: + result = await _do_create_dependency( + title="CVE-2024-0001 in sample 1.0.0", + description="Published advisory affects the pinned version.", + target="repo/package.json", + cve="CVE-2024-0001", + package_name="sample", + installed_version="1.0.0", + impact="Impact.", + remediation_steps="Upgrade.", + assumptions="Assumptions.", + package_ecosystem="npm", + fixed_version="1.0.1", + cwe=None, + advisory_cvss=5.0, + technical_analysis=None, + fix_effort="low", + reachability="not_imported", + ) + + assert result["success"] is False + assert any("reachability_evidence is required" in e for e in result["errors"]) + assert not report_state.vulnerability_reports + + +async def test_dependency_report_rejects_unknown_reachability_level( + report_state: ReportState, +) -> None: + result = await _do_create_dependency( + title="CVE-2024-0001 in sample 1.0.0", + description="Published advisory affects the pinned version.", + target="repo/package.json", + cve="CVE-2024-0001", + package_name="sample", + installed_version="1.0.0", + impact="Impact.", + remediation_steps="Upgrade.", + assumptions="Assumptions.", + package_ecosystem="npm", + fixed_version="1.0.1", + cwe=None, + advisory_cvss=5.0, + technical_analysis=None, + fix_effort="low", + reachability="not_exploitable", + reachability_evidence="vibes", + ) + + assert result["success"] is False + assert any("Invalid reachability" in e for e in result["errors"]) + assert not report_state.vulnerability_reports + + +async def test_dependency_report_omits_unknown_reachability(report_state: ReportState) -> None: + result = await _do_create_dependency( + title="CVE-2024-0001 in sample 1.0.0", + description="Published advisory affects the pinned version.", + target="repo/package.json", + cve="CVE-2024-0001", + package_name="sample", + installed_version="1.0.0", + impact="Impact.", + remediation_steps="Upgrade.", + assumptions="Analysis was inconclusive.", + package_ecosystem="npm", + fixed_version="1.0.1", + cwe=None, + advisory_cvss=5.0, + technical_analysis=None, + fix_effort="low", + ) + + assert result["success"] is True + metadata = report_state.vulnerability_reports[0]["dependency_metadata"] + assert "reachability" not in metadata + assert "reachability_evidence" not in metadata + + async def test_dependency_report_requires_advisory_cvss(report_state: ReportState) -> None: result = await _do_create_dependency( title="CVE-2024-0001 in sample 1.0.0", @@ -622,6 +736,8 @@ def test_vuln_tool_exposes_new_params() -> None: dep_props = create_dependency_report.params_json_schema["properties"] for field in ("package_name", "installed_version", "cve", "advisory_cvss"): assert field in dep_props + for field in ("reachability", "reachability_evidence"): + assert field in dep_props dep_required = create_dependency_report.params_json_schema["required"] assert "package_ecosystem" in dep_required assert "advisory_cvss" in dep_required