coverage: drop the rendered report section

The ledger stays a machine-written artifact (coverage.json and the SARIF
non-failing results); the executive report goes back to being the agent's
document.
This commit is contained in:
Ahmed Allam
2026-08-09 22:58:08 +00:00
parent 970e8ed395
commit 30f8d5fa29
7 changed files with 9 additions and 139 deletions
+1 -72
View File
@@ -18,7 +18,7 @@ in, and they are kept apart on purpose:
A coverage claim is an attestation, so conflating the two would be the worst
possible failure: a hallucinated "tested and clean" is strictly less honest
than no coverage section at all. Every entry therefore carries its ``source``,
than no coverage record at all. Every entry therefore carries its ``source``,
and machine-observed facts contradict rather than confirm — an agent that
carried the ``sql_injection`` skill and recorded nothing about SQL injection
shows up under ``gaps``, and a run that hit its budget ceiling is stamped
@@ -420,74 +420,3 @@ def write_coverage(run_dir: Path, document: dict[str, Any]) -> Path:
len(document.get("gaps", [])),
)
return path
def render_coverage_markdown(document: dict[str, Any]) -> str:
"""Render the coverage appendix appended to the executive report.
Built from the document rather than written by an agent, so the table in
the client-facing report cannot drift from the ledger it summarizes.
"""
summary = document.get("summary", {})
completeness = document.get("completeness", {})
entries = document.get("entries", [])
gaps = document.get("gaps", [])
lines = [
"# Coverage",
"",
"Surfaces assessed during this engagement and how each was closed. Entries are "
"reported by the testing agents; the completeness notes below are recorded by "
"the runtime.",
"",
f"**Surfaces reviewed:** {summary.get('surfaces_reviewed', 0)} ",
f"**Findings filed:** {summary.get('findings_filed', 0)} ",
f"**Open items and gaps:** {summary.get('gaps', 0)}",
"",
]
if entries:
lines += [
"| Surface | Risk Area | Outcome | Evidence |",
"| --- | --- | --- | --- |",
]
lines += [
"| {} | {} | {} | {} |".format(
_cell(entry.get("surface")),
_cell(entry.get("risk_area")),
_cell(entry.get("outcome_label") or entry.get("outcome")),
_cell(entry.get("evidence")),
)
for entry in entries
]
lines.append("")
else:
lines += [
"No surfaces were recorded for this scan. The absence of findings below "
"cannot be read as evidence that any particular area was tested.",
"",
]
if gaps:
lines += ["## Not Covered", ""]
lines += [f"- {_inline(gap.get('detail'))}" for gap in gaps]
lines.append("")
caveats = completeness.get("caveats") or []
if caveats:
lines += ["## Completeness", ""]
lines += [f"- {_inline(caveat)}" for caveat in caveats]
lines.append("")
return "\n".join(lines)
def _cell(value: Any) -> str:
"""Flatten *value* for a markdown table cell."""
text = _inline(value)
return text.replace("|", "\\|") if text else ""
def _inline(value: Any) -> str:
"""Collapse *value* to a single line of plain text."""
return " ".join(str(value or "").split())
+2 -6
View File
@@ -14,7 +14,7 @@ from agents.usage import Usage
from strix.config import codex
from strix.config.loader import load_settings
from strix.core.paths import run_dir_for, runtime_state_dir
from strix.report.coverage import render_coverage_markdown, write_coverage
from strix.report.coverage import write_coverage
from strix.report.sarif import write_sarif
from strix.report.usage import LLMUsageLedger
from strix.report.writer import (
@@ -469,11 +469,7 @@ class ReportState:
logger.exception("coverage.json write failed (non-fatal)")
if self.final_scan_result:
write_executive_report(
run_dir,
self.final_scan_result,
render_coverage_markdown(coverage) if coverage else None,
)
write_executive_report(run_dir, self.final_scan_result)
if self.vulnerability_reports:
write_vulnerabilities(run_dir, self.vulnerability_reports, self._saved_vuln_ids)
+5 -13
View File
@@ -113,20 +113,12 @@ def write_run_record(run_dir: Path, run_record: dict[str, Any]) -> None:
)
def write_executive_report(
run_dir: Path,
final_scan_result: str,
coverage_markdown: str | None = None,
) -> None:
def write_executive_report(run_dir: Path, final_scan_result: str) -> None:
path = run_dir / "penetration_test_report.md"
sections = [
"# Security Penetration Test Report\n",
f"**Generated:** {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S UTC')}\n",
f"{final_scan_result}\n",
]
if coverage_markdown:
sections.append(f"{coverage_markdown.rstrip()}\n")
atomic_write_text(path, "\n".join(sections))
with path.open("w", encoding="utf-8") as f:
f.write("# Security Penetration Test Report\n\n")
f.write(f"**Generated:** {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S UTC')}\n\n")
f.write(f"{final_scan_result}\n")
logger.info("Saved final penetration test report to: %s", path)
+1 -7
View File
@@ -229,13 +229,7 @@ async def finish_scan(
- ``methodology`` — frameworks followed (OWASP WSTG, PTES,
OSSTMM, NIST), engagement type (black/gray/white box), scope
and constraints, categories of testing performed. **No**
internal execution detail. Do **not** hand-write a table of
reviewed surfaces here: the report gains a ``Coverage``
section rendered directly from the coverage ledger, and a
transcribed copy would only drift from it. Describe the
approach and the classes of testing, and reconcile the ledger
via ``list_coverage`` (step 5) so that rendered section is
complete and honest.
internal execution detail.
- ``technical_analysis`` — consolidated findings overview with
severity model and systemic root causes. Reference individual
vuln reports for repro steps; don't duplicate raw evidence.
-22
View File
@@ -9,7 +9,6 @@ from strix.report.coverage import (
_SKILL_PHRASINGS,
build_coverage_document,
read_agent_graph,
render_coverage_markdown,
write_coverage,
)
from strix.skills import get_available_skills
@@ -171,27 +170,6 @@ def test_write_coverage_emits_a_top_level_artifact(tmp_path: Path) -> None:
assert json.loads(path.read_text(encoding="utf-8"))["schema_version"] == 1
def test_markdown_renders_a_surface_table() -> None:
markdown = render_coverage_markdown(_document())
assert "# Coverage" in markdown
assert "| POST /api/orders/{id} | object-level authorization | No issue identified |" in (
markdown
)
def test_markdown_says_so_when_nothing_was_recorded() -> None:
markdown = render_coverage_markdown(_document(entries=[], agent_graph={}))
assert "cannot be read as evidence" in markdown
def test_markdown_cell_escapes_pipes() -> None:
markdown = render_coverage_markdown(_document(entries=[_entry(surface="a|b")]))
assert "| a\\|b |" in markdown
def test_read_agent_graph_tolerates_a_missing_or_corrupt_snapshot(tmp_path: Path) -> None:
assert read_agent_graph(tmp_path) == {}
-8
View File
@@ -206,11 +206,3 @@ def test_render_vulnerability_md_surfaces_calibration_metadata() -> None:
assert "## Confidence Rationale" in md
assert "## What Would Change This Severity" in md
assert "## Fix Verification" in md
def test_write_executive_report_appends_the_coverage_section(tmp_path: Path) -> None:
write_executive_report(tmp_path, "Scan complete.", "# Coverage\n\nNothing tested.\n")
content = (tmp_path / "penetration_test_report.md").read_text(encoding="utf-8")
assert content.index("Scan complete.") < content.index("# Coverage")
assert "Nothing tested." in content
-11
View File
@@ -45,17 +45,6 @@ def test_coverage_is_written_beside_the_other_artifacts(state: ReportState) -> N
assert document["summary"]["surfaces_reviewed"] == 1
def test_report_carries_a_coverage_section_from_the_ledger(state: ReportState) -> None:
_record_a_cleared_surface()
state.final_scan_result = "Scan complete."
state._save_artifacts()
report = (state.get_run_dir() / "penetration_test_report.md").read_text(encoding="utf-8")
assert "# Coverage" in report
assert "POST /api/orders/{id}" in report
def test_cleared_surfaces_reach_sarif(state: ReportState) -> None:
_record_a_cleared_surface()