From a46a60cf6a78f15f556b3456ecf6452a0ad06e6f Mon Sep 17 00:00:00 2001 From: Ahmed Allam Date: Mon, 17 Aug 2026 11:34:15 +0000 Subject: [PATCH] feat(reporting): require contextual CVSS and usage evidence on dependency reports --- .../skills/custom/dependency_cve_scanning.md | 39 +++++--- strix/tools/reporting/tool.py | 55 +++++++----- tests/test_reporting_fields.py | 89 +++++++++++++++---- 3 files changed, 133 insertions(+), 50 deletions(-) diff --git a/strix/skills/custom/dependency_cve_scanning.md b/strix/skills/custom/dependency_cve_scanning.md index e5afd3de..9c5139c3 100644 --- a/strix/skills/custom/dependency_cve_scanning.md +++ b/strix/skills/custom/dependency_cve_scanning.md @@ -248,15 +248,25 @@ findings and rejects empty PoC fields): - Set `cwe` to the most specific `CWE-NNN` when the advisory names one. - Do NOT cap severity at LOW just because there is no dynamic reproduction — use the advisory score. -- Set `reachability` + `reachability_evidence` from the usage analysis above; +- Set `reachability` + `reachability_evidence` from the usage analysis above — + the tool rejects a report with no evidence, so for `unknown` write what you + searched and why the result is inconclusive; use `assumptions` for anything softer (confidence, caveats, analysis limits). -- Set `contextual_cvss_breakdown` + `contextual_cvss_reasoning` when this - codebase clearly changes the risk the published score describes (see below). +- **Always set `contextual_cvss_breakdown` + `contextual_cvss_reasoning`.** Every + dependency finding carries a contextual rating of the CVE in this codebase + (see below). Start from the published metrics and change only what your + evidence proves. +- Set every other field the report accepts when the information exists: + `package`, `ecosystem`, `installed_version`, `fixed_version`, `manifest_path`, + `introduced_by` for a transitive package, `dependency_path`, `cwe`, + `assumptions`, and the remediation instruction. A blank field costs the reader + a triage step. ### Contextual CVSS The published score rates the CVE in the abstract. `contextual_cvss_breakdown` -rates it **here**, in this codebase — the same 8-metric CVSS v3.1 object as a +rates it **here**, in this codebase, and every dependency report must carry +one. It is the same 8-metric CVSS v3.1 object as a normal finding's `cvss_breakdown` (`attack_vector`, `attack_complexity`, `privileges_required`, `user_interaction`, `scope`, `confidentiality`, `integrity`, `availability`). You never pass a score: the contextual score and @@ -285,8 +295,12 @@ come from what the source requires; `attack_complexity` comes from the preconditions the hops enforce; `confidentiality`, `integrity`, and `availability` come from the data and privileges available at the sink. -No trace, no contextual breakdown: if you did not reach a symbol hit, or you -could not follow a hop, omit the contextual fields instead of guessing. +When you have no source-to-sink trace, still rate the finding: copy the +published metrics, change only the metrics the usage level itself proves, and +say so in the reasoning. For example, for a `not_imported` package that the +build still ships, keep the published metrics and lower `confidentiality`, +`integrity`, and `availability` to `N`, because no code path reaches the +vulnerable symbol. Never invent a hop you did not read. `contextual_cvss_reasoning` is required with the breakdown. Write two to four sentences that another engineer can check without opening the repository. Name @@ -300,9 +314,10 @@ for an operator-supplied path behind the `--allow-unsafe-import` flag that attacker must already hold shell access on the job host, and the parsed data is build metadata rather than customer records." -Omit all the contextual fields when the published rating already fits, and when -the evidence is thin. A contextual rating is a claim you must be able to -defend, and it never replaces `advisory_cvss` as the published reference. +When the published rating already fits this codebase, repeat the published +metrics in the breakdown and say in the reasoning that the deployment matches +the advisory. A contextual rating is a claim you must be able to defend, and it +never replaces `advisory_cvss` as the published reference. Verify the CVE with `web_search` when available before reporting. Never guess or hallucinate a CVE id. @@ -320,5 +335,7 @@ hallucinate a CVE id. - Do not downgrade advisory severity for lack of dynamic reproduction. - Do not claim a `reachability` level the evidence does not prove — `unknown` with a reason is always acceptable; an overclaimed level never is. -- Do not send `contextual_cvss_breakdown` without evidence-backed reasoning, and - do not use it to quietly de-rate a CVE you simply could not analyze. +- Do not send a report without `contextual_cvss_breakdown` and + `contextual_cvss_reasoning` — the reader rates and ranks the finding with them. +- Do not use the contextual breakdown to quietly de-rate a CVE you could not + analyze. State the limit of the analysis in the reasoning instead. diff --git a/strix/tools/reporting/tool.py b/strix/tools/reporting/tool.py index 55d711d3..6cae75f1 100644 --- a/strix/tools/reporting/tool.py +++ b/strix/tools/reporting/tool.py @@ -757,19 +757,28 @@ def _validate_contextual_cvss( reasoning: str | None, ) -> list[str]: errors: list[str] = [] - if breakdown: + if not breakdown: + errors.append( + "contextual_cvss_breakdown is required: rate the CVE in this codebase with " + "all 8 CVSS v3.1 metrics (attack_vector, attack_complexity, " + "privileges_required, user_interaction, scope, confidentiality, integrity, " + "availability). When your trace does not change the published rating, repeat " + "the advisory's own metrics and adjust only what the usage level proves - a " + "package the code never imports is normally N on all three impact metrics." + ) + else: for name, valid in _CVSS_VALID.items(): value = breakdown.get(name) if value not in valid: errors.append( f"Invalid contextual_cvss_breakdown {name}: {value}. Must be one of: {valid}" ) - if not (reasoning or "").strip(): - errors.append( - "contextual_cvss_reasoning is required when contextual_cvss_breakdown is " - "set: state what you observed in this codebase that justifies the " - "contextual rating. A contextual score with no reasoning is not shown." - ) + if not (reasoning or "").strip(): + errors.append( + "contextual_cvss_reasoning is required: state what you observed in this " + "codebase that justifies the contextual rating. A contextual score with " + "no reasoning is not shown." + ) return errors @@ -837,9 +846,7 @@ def _build_dependency_metadata( metadata["introduced_by"] = introduced_by.strip() if dependency_path and dependency_path.strip(): metadata["dependency_path"] = dependency_path.strip() - # "unknown" is the absent case — omitting it keeps the jsonb contract clean, - # and evidence without a level would have nothing to qualify. - if reachability and reachability.strip() and reachability.strip() != "unknown": + if reachability and reachability.strip(): metadata["reachability"] = reachability.strip() if reachability_evidence and reachability_evidence.strip(): metadata["reachability_evidence"] = reachability_evidence.strip() @@ -975,11 +982,12 @@ async def _do_create_dependency( # noqa: PLR0912 errors.append( f"Invalid reachability: {reachability!r}. Must be one of: {sorted(_VALID_REACHABILITY)}" ) - elif reachability != "unknown" and not (reachability_evidence or "").strip(): + elif not (reachability_evidence or "").strip(): errors.append( - "reachability_evidence is required when reachability is not 'unknown': " - "cite the concrete proof (import file:line, matched symbol usage, or " - "govulncheck call path). Never claim a reachability level without evidence." + "reachability_evidence is required: cite the concrete proof (import " + "file:line, matched symbol usage, or govulncheck call path), or, for " + "'unknown', say what you searched and why the result is inconclusive. " + "Never claim a reachability level without evidence." ) errors.extend(_validate_contextual_cvss(contextual_cvss_breakdown, contextual_cvss_reasoning)) @@ -1217,8 +1225,9 @@ async def create_dependency_report( ``not_imported`` / ``imported`` / ``vulnerable_symbol_used`` / ``reachable_call_path`` / ``unknown``. Claim only what the evidence proves; when in doubt use ``unknown``. - reachability_evidence: The concrete proof for the claimed level - (required for any level other than ``unknown``): repo-relative + reachability_evidence: **Required.** The concrete proof for the + claimed level, or, for ``unknown``, what you searched and why + the result is inconclusive: repo-relative ``file:line`` of the import or symbol usage, the matched advisory symbols, or the govulncheck call-path excerpt. Whenever you found the vulnerable symbol in use, also give the @@ -1232,7 +1241,7 @@ async def create_dependency_report( is off in production), and say who controls the input. State it plainly when no entry point reaches the sink — that is the most useful result a reader can get. - contextual_cvss_breakdown: Optional full CVSS v3.1 rating of this + contextual_cvss_breakdown: **Required.** Full CVSS v3.1 rating of this CVE **in this codebase** — the same 8-metric object as ``create_vulnerability_report``'s ``cvss_breakdown``: ``attack_vector`` (N/A/L/P), ``attack_complexity`` (L/H), @@ -1250,11 +1259,13 @@ async def create_dependency_report( hops enforce, and the impact metrics from the data and privileges reachable at the sink. When provided, this rating determines the finding's severity; ``advisory_cvss`` stays as - the published reference. Omit the field when the trace does - not change the published rating, or when you could not - complete the trace. - contextual_cvss_reasoning: **Required whenever** - ``contextual_cvss_breakdown`` is set. Two to four detailed + the published reference. Send it on every report: when the + trace does not change the published rating, or when you could + not complete the trace, repeat the advisory's own metrics and + adjust only what the usage level itself proves (a package the + code never imports is normally ``N`` on all three impact + metrics), then say so in the reasoning. + contextual_cvss_reasoning: **Required.** Two to four detailed sentences that a reviewer can verify without opening the repo: how the application uses the package, which call sites or configuration you inspected (repo-relative ``file:line``), diff --git a/tests/test_reporting_fields.py b/tests/test_reporting_fields.py index d19f7aec..d433db40 100644 --- a/tests/test_reporting_fields.py +++ b/tests/test_reporting_fields.py @@ -37,6 +37,24 @@ _CVSS = { } +_DEP_CONTEXT = { + "attack_vector": "N", + "attack_complexity": "L", + "privileges_required": "N", + "user_interaction": "N", + "scope": "U", + "confidentiality": "N", + "integrity": "N", + "availability": "H", +} + +_DEP_CONTEXT_VECTOR = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + +_DEP_EVIDENCE = "src/render.ts:14 imports the package." + +_DEP_REASONING = "Only scripts/import.py reaches the sink, so the impact is availability only." + + @pytest.fixture def report_state(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> ReportState: monkeypatch.chdir(tmp_path) @@ -147,13 +165,17 @@ async def test_dependency_report_sets_class_and_metadata(report_state: ReportSta advisory_cvss=7.2, technical_analysis=None, fix_effort="trivial", + reachability="imported", + reachability_evidence=_DEP_EVIDENCE, + contextual_cvss_breakdown=_DEP_CONTEXT, + contextual_cvss_reasoning=_DEP_REASONING, ) assert result["success"] is True report = report_state.vulnerability_reports[0] assert report["finding_class"] == "dependency_cve" assert report["cve"] == "CVE-2021-23337" assert report["severity"] == "high" - assert report["evidence"] == ( + assert report["evidence"].startswith( "**Advisory evidence:** `CVE-2021-23337` applies to `lodash` " "at installed version `4.17.20`. The advisory is fixed in `4.17.21`." ) @@ -164,6 +186,12 @@ async def test_dependency_report_sets_class_and_metadata(report_state: ReportSta "package_ecosystem": "npm", "manifest_path": "package-lock.json", "fixed_version": "4.17.21", + "reachability": "imported", + "reachability_evidence": _DEP_EVIDENCE, + "contextual_cvss_breakdown": _DEP_CONTEXT, + "contextual_cvss_score": pytest.approx(7.5, abs=0.05), + "contextual_cvss_vector": _DEP_CONTEXT_VECTOR, + "contextual_cvss_reasoning": _DEP_REASONING, } @@ -187,6 +215,10 @@ async def test_dependency_report_records_transitive_chain(report_state: ReportSt fix_effort="trivial", introduced_by="express@4.18.1", dependency_path="express@4.18.1 > body-parser@1.20.0 > qs@6.10.2", + reachability="imported", + reachability_evidence=_DEP_EVIDENCE, + contextual_cvss_breakdown=_DEP_CONTEXT, + contextual_cvss_reasoning=_DEP_REASONING, ) assert result["success"] is True report = report_state.vulnerability_reports[0] @@ -225,6 +257,10 @@ async def test_dependency_report_omits_blank_chain_fields(report_state: ReportSt fix_effort="trivial", introduced_by=" ", dependency_path=None, + reachability="imported", + reachability_evidence=_DEP_EVIDENCE, + contextual_cvss_breakdown=_DEP_CONTEXT, + contextual_cvss_reasoning=_DEP_REASONING, ) assert result["success"] is True report = report_state.vulnerability_reports[0] @@ -232,7 +268,7 @@ async def test_dependency_report_omits_blank_chain_fields(report_state: ReportSt assert "dependency_path" not in report["dependency_metadata"] -async def test_dependency_report_with_zero_cvss_remains_low_severity( +async def test_dependency_report_with_no_contextual_impact_is_info( report_state: ReportState, ) -> None: result = await _do_create_dependency( @@ -252,12 +288,16 @@ async def test_dependency_report_with_zero_cvss_remains_low_severity( advisory_cvss=0.0, technical_analysis=None, fix_effort="low", + reachability="not_imported", + reachability_evidence="No file imports the package.", + contextual_cvss_breakdown={**_DEP_CONTEXT, "availability": "N"}, + contextual_cvss_reasoning="No application code imports the package.", ) assert result["success"] is True - assert result["severity"] == "low" + assert result["severity"] == "info" report = report_state.vulnerability_reports[0] - assert report["severity"] == "low" + assert report["severity"] == "info" assert report["cvss"] == 0.0 @@ -281,6 +321,8 @@ async def test_dependency_report_records_reachability(report_state: ReportState) fix_effort="low", reachability="vulnerable_symbol_used", reachability_evidence="src/render.ts:14 calls `_.template()`.", + contextual_cvss_breakdown=_DEP_CONTEXT, + contextual_cvss_reasoning=_DEP_REASONING, ) assert result["success"] is True @@ -292,7 +334,8 @@ async def test_dependency_report_records_reachability(report_state: ReportState) ) assert "**Usage analysis:**" in report["evidence"] assert "not a proof of exploitability or of safety" in report["evidence"] - # The level must never influence the rating — that stays advisory_cvss only. + # The level must never influence the rating — that comes from the contextual + # breakdown, or from advisory_cvss when no breakdown applies. assert report["severity"] == "high" @@ -353,7 +396,7 @@ async def test_dependency_report_rejects_unknown_reachability_level( assert not report_state.vulnerability_reports -async def test_dependency_report_omits_unknown_reachability(report_state: ReportState) -> None: +async def test_dependency_report_records_unknown_reachability(report_state: ReportState) -> None: result = await _do_create_dependency( title="CVE-2024-0001 in sample 1.0.0", description="Published advisory affects the pinned version.", @@ -371,12 +414,15 @@ async def test_dependency_report_omits_unknown_reachability(report_state: Report advisory_cvss=5.0, technical_analysis=None, fix_effort="low", + reachability_evidence="Grep for the package found no import.", + contextual_cvss_breakdown=_DEP_CONTEXT, + contextual_cvss_reasoning=_DEP_REASONING, ) assert result["success"] is True, result metadata = report_state.vulnerability_reports[0]["dependency_metadata"] - assert "reachability" not in metadata - assert "reachability_evidence" not in metadata + assert metadata["reachability"] == "unknown" + assert metadata["reachability_evidence"] == "Grep for the package found no import." async def test_dependency_report_requires_advisory_cvss(report_state: ReportState) -> None: @@ -453,6 +499,10 @@ async def test_dependency_report_dedupe_candidate_includes_dependency_metadata( advisory_cvss=0.0, technical_analysis=None, fix_effort="low", + reachability="imported", + reachability_evidence=_DEP_EVIDENCE, + contextual_cvss_breakdown=_DEP_CONTEXT, + contextual_cvss_reasoning=_DEP_REASONING, ) assert result["success"] is True @@ -468,6 +518,12 @@ async def test_dependency_report_dedupe_candidate_includes_dependency_metadata( "package_ecosystem": "npm", "manifest_path": "package-lock.json", "fixed_version": "1.0.1", + "reachability": "imported", + "reachability_evidence": _DEP_EVIDENCE, + "contextual_cvss_breakdown": _DEP_CONTEXT, + "contextual_cvss_score": pytest.approx(7.5, abs=0.05), + "contextual_cvss_vector": _DEP_CONTEXT_VECTOR, + "contextual_cvss_reasoning": _DEP_REASONING, }, "technical_analysis": None, } @@ -926,6 +982,8 @@ async def test_dependency_report_computes_contextual_cvss( cwe="CWE-94", fix_effort="trivial", manifest_path="package-lock.json", + reachability="vulnerable_symbol_used", + reachability_evidence="scripts/import.py:88 calls `_.template()`.", contextual_cvss_breakdown=_CONTEXTUAL_BREAKDOWN, contextual_cvss_reasoning="Only scripts/import.py reaches the sink.", ) @@ -944,7 +1002,7 @@ async def test_dependency_report_computes_contextual_cvss( @pytest.mark.asyncio -async def test_dependency_report_rates_from_advisory_without_contextual( +async def test_dependency_report_requires_contextual_breakdown( report_state: ReportState, ) -> None: result = await _do_create_dependency( @@ -964,15 +1022,12 @@ async def test_dependency_report_rates_from_advisory_without_contextual( cwe="CWE-94", fix_effort="trivial", manifest_path="package-lock.json", + reachability="imported", + reachability_evidence=_DEP_EVIDENCE, ) - assert result["success"] is True, result - report = report_state.vulnerability_reports[0] - assert report["cvss"] == 7.2 - assert report["severity"] == "high" - metadata = report["dependency_metadata"] - assert metadata["advisory_cvss"] == 7.2 - assert "contextual_cvss_breakdown" not in metadata - assert "contextual_cvss_score" not in metadata + assert result["success"] is False + assert any("contextual_cvss_breakdown is required" in error for error in result["errors"]) + assert report_state.vulnerability_reports == [] @pytest.mark.asyncio