From 310f310e280e802441bffb18a374aef3a2c8da2e Mon Sep 17 00:00:00 2001 From: Ahmed Allam Date: Mon, 17 Aug 2026 09:05:20 +0000 Subject: [PATCH] feat(reporting): contextual CVSS environmental metrics on dependency reports --- .../skills/custom/dependency_cve_scanning.md | 63 +++++++++ strix/tools/reporting/tool.py | 126 +++++++++++++++++- tests/test_reporting_fields.py | 75 ++++++++++- 3 files changed, 261 insertions(+), 3 deletions(-) diff --git a/strix/skills/custom/dependency_cve_scanning.md b/strix/skills/custom/dependency_cve_scanning.md index 129f4a1e..125b92d8 100644 --- a/strix/skills/custom/dependency_cve_scanning.md +++ b/strix/skills/custom/dependency_cve_scanning.md @@ -234,6 +234,67 @@ findings and rejects empty PoC fields): the advisory score. - Set `reachability` + `reachability_evidence` from the usage analysis above; use `assumptions` for anything softer (confidence, caveats, analysis limits). +- Set `contextual_cvss_metrics` + `contextual_cvss_reasoning` when this codebase + clearly changes the risk the published score describes (see below). + +### Contextual (environmental) CVSS + +The published score rates the CVE in the abstract. `contextual_cvss_metrics` +rates it **here**, in this codebase, with CVSS v3.1 environmental metrics. The +advisory's base metrics stay fixed — you never restate them and never pass a +score, the adjusted score is computed from the resulting vector. + +Set only what your usage analysis supports: + +- `MAV` `N`/`A`/`L`/`P` — the attack vector as deployed. A library reached only + by a local CLI is `L`, not `N`. +- `MAC` `L`/`H` — raise to `H` when the vulnerable path needs a precondition the + code enforces (input validation, a non-default flag, an internal-only route). +- `MPR` `N`/`L`/`H`, `MUI` `N`/`R` — privileges or interaction this deployment + requires before the path is reachable. +- `MS` `U`/`C` — whether exploitation here escapes the component boundary. +- `MC`/`MI`/`MA` `H`/`L`/`N` — the impact in this codebase. `not_imported` code + the build still ships is usually `N` across all three. +- `CR`/`IR`/`AR` `H`/`M`/`L` — the security requirement of the data or service + the package handles (credentials or payment data raise `CR`). + +Ground every metric in a **source-to-sink trace**, not in a general impression +of the package. Before you set any metric: + +1. Find the sink: the exact line where this codebase calls the vulnerable + function or class of the package. +2. Walk backwards hop by hop to the source: the entry point that carries + untrusted input (HTTP route, CLI argument, queue or webhook payload, + uploaded file, config value). Read each intermediate function. When a hop + is a thin wrapper, go one step deeper — never stop at the first caller. +3. Record what each hop enforces: authentication, a role check, validation, a + feature flag, a size or type limit, a default that is off in production. +4. Derive the metrics from that chain. `MAV`, `MPR`, and `MUI` come from what + the source requires. `MAC` comes from the preconditions on the hops. `MC`, + `MI`, and `MA` come from the data and privileges available at the sink. + `CR`, `IR`, and `AR` come from what that data is worth. + +If the chain breaks — no source reaches the sink, or you cannot follow a hop — +say so and omit the contextual fields instead of guessing. + +`contextual_cvss_reasoning` is required with the metrics. Write two to four +sentences that another engineer can check without opening the repository. Name +the chain hop by hop as `entry point -> intermediate call -> package call`, with +a repository-relative `file:line` for each hop, say who controls the input, and +say what the adjustment changes. Example: `{"MAC": "H", "MC": "L"}` with "The +only caller of `yaml.load` is `parse_manifest` in `scripts/import.py:88`, which +`cli/commands.py:212` invokes for an operator-supplied path behind the +`--allow-unsafe-import` flag that `deploy/prod.yaml` never sets. No HTTP route +reaches that function, so an attacker must already hold shell access on the job +host, and the parsed data is build metadata rather than customer records." + +`contextual_cvss_metric_reasoning` is optional and takes one detailed sentence +per metric you adjusted, keyed by the metric name. Use it for the per-metric +detail that does not fit the summary. + +Omit all the contextual fields when the published rating already fits, and when +the evidence is thin. A contextual score is a claim you must be able to defend, +and this adjustment never replaces `advisory_cvss`. Verify the CVE with `web_search` when available before reporting. Never guess or hallucinate a CVE id. @@ -251,3 +312,5 @@ hallucinate a CVE id. - Do not downgrade advisory severity for lack of dynamic reproduction. - Do not claim a `reachability` level the evidence does not prove — `unknown` with a reason is always acceptable; an overclaimed level never is. +- Do not send `contextual_cvss_metrics` without evidence-backed reasoning, and + do not use it to quietly de-rate a CVE you simply could not analyze. diff --git a/strix/tools/reporting/tool.py b/strix/tools/reporting/tool.py index 12dd2046..f1255eff 100644 --- a/strix/tools/reporting/tool.py +++ b/strix/tools/reporting/tool.py @@ -749,6 +749,54 @@ def _validate_manifest_path(manifest_path: str | None) -> str | None: return None +# CVSS v3.1 environmental metrics an agent may set on a dependency finding, +# with their legal values. The base metrics are deliberately absent: they come +# from the published advisory, so a report can never restate them. +_CVSS_ENVIRONMENTAL_VALUES: dict[str, frozenset[str]] = { + "MAV": frozenset("NALP"), + "MAC": frozenset("LH"), + "MPR": frozenset("NLH"), + "MUI": frozenset("NR"), + "MS": frozenset("UC"), + "MC": frozenset("HLN"), + "MI": frozenset("HLN"), + "MA": frozenset("HLN"), + "CR": frozenset("HML"), + "IR": frozenset("HML"), + "AR": frozenset("HML"), +} +_MAX_CONTEXTUAL_REASONING_CHARS = 2000 + + +def _clean_contextual_cvss_metrics(raw: dict[str, str] | None) -> dict[str, str]: + """Keep only well-formed CVSS environmental metrics from a report.""" + if not isinstance(raw, dict): + return {} + metrics: dict[str, str] = {} + for key, value in raw.items(): + metric = str(key or "").strip().upper() + metric_value = str(value or "").strip().upper() + allowed = _CVSS_ENVIRONMENTAL_VALUES.get(metric) + if allowed and metric_value in allowed: + metrics[metric] = metric_value + return metrics + + +def _clean_contextual_cvss_metric_reasoning( + raw: dict[str, str] | None, metrics: dict[str, str] +) -> dict[str, str]: + """Keep per-metric justifications that belong to an adjusted metric.""" + if not isinstance(raw, dict): + return {} + detail: dict[str, str] = {} + for key, value in raw.items(): + metric = str(key or "").strip().upper() + text = str(value or "").strip() + if metric in metrics and text: + detail[metric] = text[:_MAX_CONTEXTUAL_REASONING_CHARS] + return detail + + def _build_dependency_metadata( *, package_name: str, @@ -760,8 +808,11 @@ def _build_dependency_metadata( manifest_path: str | None = None, reachability: str | None = None, reachability_evidence: str | None = None, -) -> dict[str, str]: - metadata = { + contextual_cvss_metrics: dict[str, str] | None = None, + contextual_cvss_reasoning: str | None = None, + contextual_cvss_metric_reasoning: dict[str, str] | None = None, +) -> dict[str, Any]: + metadata: dict[str, Any] = { "package_name": package_name.strip(), "installed_version": installed_version.strip(), } @@ -781,6 +832,18 @@ def _build_dependency_metadata( metadata["reachability"] = reachability.strip() if reachability_evidence and reachability_evidence.strip(): metadata["reachability_evidence"] = reachability_evidence.strip() + # Contextual CVSS is only meaningful as metrics plus the reasoning a reader + # can check, so an incomplete pair is dropped. + cleaned_metrics = _clean_contextual_cvss_metrics(contextual_cvss_metrics) + reasoning = str(contextual_cvss_reasoning or "").strip() + if cleaned_metrics and reasoning: + metadata["contextual_cvss_metrics"] = cleaned_metrics + metadata["contextual_cvss_reasoning"] = reasoning[:_MAX_CONTEXTUAL_REASONING_CHARS] + metric_reasoning = _clean_contextual_cvss_metric_reasoning( + contextual_cvss_metric_reasoning, cleaned_metrics + ) + if metric_reasoning: + metadata["contextual_cvss_metric_reasoning"] = metric_reasoning return metadata @@ -852,6 +915,9 @@ async def _do_create_dependency( # noqa: PLR0912 manifest_path: str | None = None, reachability: str = "unknown", reachability_evidence: str | None = None, + contextual_cvss_metrics: dict[str, str] | None = None, + contextual_cvss_reasoning: str | None = None, + contextual_cvss_metric_reasoning: dict[str, str] | None = None, agent_id: str | None = None, agent_name: str | None = None, ) -> dict[str, Any]: @@ -904,6 +970,13 @@ async def _do_create_dependency( # noqa: PLR0912 "govulncheck call path). Never claim a reachability level without evidence." ) + if contextual_cvss_metrics and not (contextual_cvss_reasoning or "").strip(): + errors.append( + "contextual_cvss_reasoning is required when contextual_cvss_metrics is set: " + "state in one or two sentences what you observed in this codebase that " + "justifies the adjustment. An adjusted score with no reasoning is not shown." + ) + if advisory_cvss is None: errors.append( "advisory_cvss is required: read the published advisory base score " @@ -927,6 +1000,9 @@ async def _do_create_dependency( # noqa: PLR0912 manifest_path=manifest_path, reachability=reachability, reachability_evidence=reachability_evidence, + contextual_cvss_metrics=contextual_cvss_metrics, + contextual_cvss_reasoning=contextual_cvss_reasoning, + contextual_cvss_metric_reasoning=contextual_cvss_metric_reasoning, ) evidence = _build_dependency_evidence( cve=parsed_cve, @@ -1038,6 +1114,9 @@ async def create_dependency_report( dependency_path: str | None = None, reachability: str = "unknown", reachability_evidence: str | None = None, + contextual_cvss_metrics: dict[str, str] | None = None, + contextual_cvss_reasoning: str | None = None, + contextual_cvss_metric_reasoning: dict[str, str] | None = None, ) -> str: """File a known-CVE dependency (SCA) finding — one report per CVE x package. @@ -1131,6 +1210,46 @@ async def create_dependency_report( (required for any level other than ``unknown``): repo-relative ``file:line`` of the import or symbol usage, the matched advisory symbols, or the govulncheck call-path excerpt. + contextual_cvss_metrics: Optional CVSS v3.1 **environmental** + metrics that reframe the published score for this codebase, + as a mapping of metric to value: ``MAV`` (N/A/L/P), ``MAC`` + (L/H), ``MPR`` (N/L/H), ``MUI`` (N/R), ``MS`` (U/C), ``MC`` / + ``MI`` / ``MA`` (H/L/N), ``CR`` / ``IR`` / ``AR`` (H/M/L). + Set only the metrics your evidence supports (for example + ``{"MAC": "H", "MC": "L"}`` when the vulnerable path needs a + precondition this deployment enforces and the data at risk is + limited). Base the values on a **source-to-sink trace**: start + at the entry point that carries untrusted input (HTTP route, + CLI argument, queue message, webhook, config file), follow + each hop of the data through this codebase, and end at the + vulnerable package call site. Go one step deeper whenever a + hop is a wrapper — never stop at the first caller. Adjust + ``MAV`` / ``MPR`` / ``MUI`` from what that entry point + actually requires, and ``MC`` / ``MI`` / ``MA`` from the data + and privileges reachable at the sink. You never supply base + metrics or a score: the base vector comes from the advisory + and the adjusted score is computed from the resulting vector. + Omit the field when the trace does not change the published + rating, or when you could not complete the trace. + contextual_cvss_reasoning: **Required whenever** + ``contextual_cvss_metrics`` is set. Two to four detailed + sentences that a reviewer can verify without opening the repo: + how the application uses the package, which call sites or + configuration you inspected (repo-relative ``file:line``), + which input reaches the vulnerable code and whether an + attacker controls it, and what the adjustment therefore + changes. State the source-to-sink chain explicitly, hop by + hop, as ``entry point -> intermediate call -> package call`` + with a ``file:line`` for each hop. Cite concrete evidence, + never a generic statement such as "low risk". The user reads + this text next to the adjusted score, so an adjustment + without it is discarded. + contextual_cvss_metric_reasoning: Optional per-metric detail, as a + mapping of the SAME metric names you set in + ``contextual_cvss_metrics`` to one detailed sentence each + (for example ``{"MAC": "Reaching the parser needs the + --unsafe flag, which deploy/prod.yaml never sets."}``). + Entries for metrics you did not adjust are dropped. """ agent_id, agent_name = _caller_identity(ctx) @@ -1155,6 +1274,9 @@ async def create_dependency_report( manifest_path=manifest_path, reachability=reachability, reachability_evidence=reachability_evidence, + contextual_cvss_metrics=contextual_cvss_metrics, + contextual_cvss_reasoning=contextual_cvss_reasoning, + contextual_cvss_metric_reasoning=contextual_cvss_metric_reasoning, agent_id=agent_id, agent_name=agent_name, ) diff --git a/tests/test_reporting_fields.py b/tests/test_reporting_fields.py index b52fe4fd..d698c904 100644 --- a/tests/test_reporting_fields.py +++ b/tests/test_reporting_fields.py @@ -372,7 +372,7 @@ async def test_dependency_report_omits_unknown_reachability(report_state: Report fix_effort="low", ) - assert result["success"] is True + assert result["success"] is True, result metadata = report_state.vulnerability_reports[0]["dependency_metadata"] assert "reachability" not in metadata assert "reachability_evidence" not in metadata @@ -877,3 +877,76 @@ def test_vuln_tool_exposes_new_params() -> None: dep_required = create_dependency_report.params_json_schema["required"] assert "package_ecosystem" in dep_required assert "advisory_cvss" in dep_required + + +def test_dep_tool_exposes_contextual_cvss_params() -> None: + dep_props = create_dependency_report.params_json_schema["properties"] + for field in ( + "contextual_cvss_metrics", + "contextual_cvss_reasoning", + "contextual_cvss_metric_reasoning", + ): + assert field in dep_props + assert "source-to-sink" in dep_props["contextual_cvss_metrics"]["description"].lower() + assert "file:line" in dep_props["contextual_cvss_reasoning"]["description"].lower() + + +@pytest.mark.asyncio +async def test_dependency_report_keeps_only_valid_contextual_metrics( + report_state: ReportState, +) -> None: + result = await _do_create_dependency( + title="CVE-2021-23337 in lodash 4.17.20", + description="Command injection via template.", + target="repo/package.json", + cve="CVE-2021-23337", + package_name="lodash", + installed_version="4.17.20", + impact="Arbitrary command execution.", + remediation_steps="Upgrade to 4.17.21.", + assumptions="Assumes the template sink is reachable.", + package_ecosystem="npm", + advisory_cvss=7.2, + technical_analysis=None, + fixed_version="4.17.21", + cwe="CWE-94", + fix_effort="trivial", + manifest_path="package-lock.json", + contextual_cvss_metrics={"MAC": "H", "MC": "L", "AV": "N", "MPR": "Z"}, + contextual_cvss_reasoning="Only scripts/import.py reaches the sink.", + contextual_cvss_metric_reasoning={"MAC": "Needs a build flag.", "MPR": "dropped"}, + ) + assert result["success"] is True, result + metadata = report_state.vulnerability_reports[0]["dependency_metadata"] + assert metadata["contextual_cvss_metrics"] == {"MAC": "H", "MC": "L"} + assert metadata["contextual_cvss_reasoning"] == "Only scripts/import.py reaches the sink." + assert metadata["contextual_cvss_metric_reasoning"] == {"MAC": "Needs a build flag."} + + +@pytest.mark.asyncio +async def test_dependency_report_rejects_contextual_metrics_without_reasoning( + report_state: ReportState, +) -> None: + result = await _do_create_dependency( + title="CVE-2021-23337 in lodash 4.17.20", + description="Command injection via template.", + target="repo/package.json", + cve="CVE-2021-23337", + package_name="lodash", + installed_version="4.17.20", + impact="Arbitrary command execution.", + remediation_steps="Upgrade to 4.17.21.", + assumptions="Assumes the template sink is reachable.", + package_ecosystem="npm", + advisory_cvss=7.2, + technical_analysis=None, + fixed_version="4.17.21", + cwe="CWE-94", + fix_effort="trivial", + manifest_path="package-lock.json", + contextual_cvss_metrics={"MAC": "H"}, + contextual_cvss_reasoning=" ", + ) + assert result["success"] is False + assert any("contextual_cvss_reasoning is required" in error for error in result["errors"]) + assert report_state.vulnerability_reports == []