mirror of
https://github.com/usestrix/strix.git
synced 2026-08-16 09:26:39 +02:00
coverage: drop the rendered report section
The ledger stays a machine-written artifact (coverage.json and the SARIF non-failing results); the executive report goes back to being the agent's document.
This commit is contained in:
@@ -18,7 +18,7 @@ in, and they are kept apart on purpose:
|
||||
|
||||
A coverage claim is an attestation, so conflating the two would be the worst
|
||||
possible failure: a hallucinated "tested and clean" is strictly less honest
|
||||
than no coverage section at all. Every entry therefore carries its ``source``,
|
||||
than no coverage record at all. Every entry therefore carries its ``source``,
|
||||
and machine-observed facts contradict rather than confirm — an agent that
|
||||
carried the ``sql_injection`` skill and recorded nothing about SQL injection
|
||||
shows up under ``gaps``, and a run that hit its budget ceiling is stamped
|
||||
@@ -420,74 +420,3 @@ def write_coverage(run_dir: Path, document: dict[str, Any]) -> Path:
|
||||
len(document.get("gaps", [])),
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def render_coverage_markdown(document: dict[str, Any]) -> str:
|
||||
"""Render the coverage appendix appended to the executive report.
|
||||
|
||||
Built from the document rather than written by an agent, so the table in
|
||||
the client-facing report cannot drift from the ledger it summarizes.
|
||||
"""
|
||||
summary = document.get("summary", {})
|
||||
completeness = document.get("completeness", {})
|
||||
entries = document.get("entries", [])
|
||||
gaps = document.get("gaps", [])
|
||||
|
||||
lines = [
|
||||
"# Coverage",
|
||||
"",
|
||||
"Surfaces assessed during this engagement and how each was closed. Entries are "
|
||||
"reported by the testing agents; the completeness notes below are recorded by "
|
||||
"the runtime.",
|
||||
"",
|
||||
f"**Surfaces reviewed:** {summary.get('surfaces_reviewed', 0)} ",
|
||||
f"**Findings filed:** {summary.get('findings_filed', 0)} ",
|
||||
f"**Open items and gaps:** {summary.get('gaps', 0)}",
|
||||
"",
|
||||
]
|
||||
|
||||
if entries:
|
||||
lines += [
|
||||
"| Surface | Risk Area | Outcome | Evidence |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
lines += [
|
||||
"| {} | {} | {} | {} |".format(
|
||||
_cell(entry.get("surface")),
|
||||
_cell(entry.get("risk_area")),
|
||||
_cell(entry.get("outcome_label") or entry.get("outcome")),
|
||||
_cell(entry.get("evidence")),
|
||||
)
|
||||
for entry in entries
|
||||
]
|
||||
lines.append("")
|
||||
else:
|
||||
lines += [
|
||||
"No surfaces were recorded for this scan. The absence of findings below "
|
||||
"cannot be read as evidence that any particular area was tested.",
|
||||
"",
|
||||
]
|
||||
|
||||
if gaps:
|
||||
lines += ["## Not Covered", ""]
|
||||
lines += [f"- {_inline(gap.get('detail'))}" for gap in gaps]
|
||||
lines.append("")
|
||||
|
||||
caveats = completeness.get("caveats") or []
|
||||
if caveats:
|
||||
lines += ["## Completeness", ""]
|
||||
lines += [f"- {_inline(caveat)}" for caveat in caveats]
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _cell(value: Any) -> str:
|
||||
"""Flatten *value* for a markdown table cell."""
|
||||
text = _inline(value)
|
||||
return text.replace("|", "\\|") if text else "—"
|
||||
|
||||
|
||||
def _inline(value: Any) -> str:
|
||||
"""Collapse *value* to a single line of plain text."""
|
||||
return " ".join(str(value or "").split())
|
||||
|
||||
@@ -14,7 +14,7 @@ from agents.usage import Usage
|
||||
from strix.config import codex
|
||||
from strix.config.loader import load_settings
|
||||
from strix.core.paths import run_dir_for, runtime_state_dir
|
||||
from strix.report.coverage import render_coverage_markdown, write_coverage
|
||||
from strix.report.coverage import write_coverage
|
||||
from strix.report.sarif import write_sarif
|
||||
from strix.report.usage import LLMUsageLedger
|
||||
from strix.report.writer import (
|
||||
@@ -469,11 +469,7 @@ class ReportState:
|
||||
logger.exception("coverage.json write failed (non-fatal)")
|
||||
|
||||
if self.final_scan_result:
|
||||
write_executive_report(
|
||||
run_dir,
|
||||
self.final_scan_result,
|
||||
render_coverage_markdown(coverage) if coverage else None,
|
||||
)
|
||||
write_executive_report(run_dir, self.final_scan_result)
|
||||
|
||||
if self.vulnerability_reports:
|
||||
write_vulnerabilities(run_dir, self.vulnerability_reports, self._saved_vuln_ids)
|
||||
|
||||
+5
-13
@@ -113,20 +113,12 @@ def write_run_record(run_dir: Path, run_record: dict[str, Any]) -> None:
|
||||
)
|
||||
|
||||
|
||||
def write_executive_report(
|
||||
run_dir: Path,
|
||||
final_scan_result: str,
|
||||
coverage_markdown: str | None = None,
|
||||
) -> None:
|
||||
def write_executive_report(run_dir: Path, final_scan_result: str) -> None:
|
||||
path = run_dir / "penetration_test_report.md"
|
||||
sections = [
|
||||
"# Security Penetration Test Report\n",
|
||||
f"**Generated:** {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S UTC')}\n",
|
||||
f"{final_scan_result}\n",
|
||||
]
|
||||
if coverage_markdown:
|
||||
sections.append(f"{coverage_markdown.rstrip()}\n")
|
||||
atomic_write_text(path, "\n".join(sections))
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write("# Security Penetration Test Report\n\n")
|
||||
f.write(f"**Generated:** {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S UTC')}\n\n")
|
||||
f.write(f"{final_scan_result}\n")
|
||||
logger.info("Saved final penetration test report to: %s", path)
|
||||
|
||||
|
||||
|
||||
@@ -229,13 +229,7 @@ async def finish_scan(
|
||||
- ``methodology`` — frameworks followed (OWASP WSTG, PTES,
|
||||
OSSTMM, NIST), engagement type (black/gray/white box), scope
|
||||
and constraints, categories of testing performed. **No**
|
||||
internal execution detail. Do **not** hand-write a table of
|
||||
reviewed surfaces here: the report gains a ``Coverage``
|
||||
section rendered directly from the coverage ledger, and a
|
||||
transcribed copy would only drift from it. Describe the
|
||||
approach and the classes of testing, and reconcile the ledger
|
||||
via ``list_coverage`` (step 5) so that rendered section is
|
||||
complete and honest.
|
||||
internal execution detail.
|
||||
- ``technical_analysis`` — consolidated findings overview with
|
||||
severity model and systemic root causes. Reference individual
|
||||
vuln reports for repro steps; don't duplicate raw evidence.
|
||||
|
||||
@@ -9,7 +9,6 @@ from strix.report.coverage import (
|
||||
_SKILL_PHRASINGS,
|
||||
build_coverage_document,
|
||||
read_agent_graph,
|
||||
render_coverage_markdown,
|
||||
write_coverage,
|
||||
)
|
||||
from strix.skills import get_available_skills
|
||||
@@ -171,27 +170,6 @@ def test_write_coverage_emits_a_top_level_artifact(tmp_path: Path) -> None:
|
||||
assert json.loads(path.read_text(encoding="utf-8"))["schema_version"] == 1
|
||||
|
||||
|
||||
def test_markdown_renders_a_surface_table() -> None:
|
||||
markdown = render_coverage_markdown(_document())
|
||||
|
||||
assert "# Coverage" in markdown
|
||||
assert "| POST /api/orders/{id} | object-level authorization | No issue identified |" in (
|
||||
markdown
|
||||
)
|
||||
|
||||
|
||||
def test_markdown_says_so_when_nothing_was_recorded() -> None:
|
||||
markdown = render_coverage_markdown(_document(entries=[], agent_graph={}))
|
||||
|
||||
assert "cannot be read as evidence" in markdown
|
||||
|
||||
|
||||
def test_markdown_cell_escapes_pipes() -> None:
|
||||
markdown = render_coverage_markdown(_document(entries=[_entry(surface="a|b")]))
|
||||
|
||||
assert "| a\\|b |" in markdown
|
||||
|
||||
|
||||
def test_read_agent_graph_tolerates_a_missing_or_corrupt_snapshot(tmp_path: Path) -> None:
|
||||
assert read_agent_graph(tmp_path) == {}
|
||||
|
||||
|
||||
@@ -206,11 +206,3 @@ def test_render_vulnerability_md_surfaces_calibration_metadata() -> None:
|
||||
assert "## Confidence Rationale" in md
|
||||
assert "## What Would Change This Severity" in md
|
||||
assert "## Fix Verification" in md
|
||||
|
||||
|
||||
def test_write_executive_report_appends_the_coverage_section(tmp_path: Path) -> None:
|
||||
write_executive_report(tmp_path, "Scan complete.", "# Coverage\n\nNothing tested.\n")
|
||||
content = (tmp_path / "penetration_test_report.md").read_text(encoding="utf-8")
|
||||
|
||||
assert content.index("Scan complete.") < content.index("# Coverage")
|
||||
assert "Nothing tested." in content
|
||||
|
||||
@@ -45,17 +45,6 @@ def test_coverage_is_written_beside_the_other_artifacts(state: ReportState) -> N
|
||||
assert document["summary"]["surfaces_reviewed"] == 1
|
||||
|
||||
|
||||
def test_report_carries_a_coverage_section_from_the_ledger(state: ReportState) -> None:
|
||||
_record_a_cleared_surface()
|
||||
state.final_scan_result = "Scan complete."
|
||||
|
||||
state._save_artifacts()
|
||||
|
||||
report = (state.get_run_dir() / "penetration_test_report.md").read_text(encoding="utf-8")
|
||||
assert "# Coverage" in report
|
||||
assert "POST /api/orders/{id}" in report
|
||||
|
||||
|
||||
def test_cleared_surfaces_reach_sarif(state: ReportState) -> None:
|
||||
_record_a_cleared_surface()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user