mirror of
https://github.com/usestrix/strix.git
synced 2026-08-25 20:32:38 +02:00
coverage: drop the rendered report section
The ledger stays a machine-written artifact (coverage.json and the SARIF non-failing results); the executive report goes back to being the agent's document.
This commit is contained in:
@@ -18,7 +18,7 @@ in, and they are kept apart on purpose:
|
|||||||
|
|
||||||
A coverage claim is an attestation, so conflating the two would be the worst
|
A coverage claim is an attestation, so conflating the two would be the worst
|
||||||
possible failure: a hallucinated "tested and clean" is strictly less honest
|
possible failure: a hallucinated "tested and clean" is strictly less honest
|
||||||
than no coverage section at all. Every entry therefore carries its ``source``,
|
than no coverage record at all. Every entry therefore carries its ``source``,
|
||||||
and machine-observed facts contradict rather than confirm — an agent that
|
and machine-observed facts contradict rather than confirm — an agent that
|
||||||
carried the ``sql_injection`` skill and recorded nothing about SQL injection
|
carried the ``sql_injection`` skill and recorded nothing about SQL injection
|
||||||
shows up under ``gaps``, and a run that hit its budget ceiling is stamped
|
shows up under ``gaps``, and a run that hit its budget ceiling is stamped
|
||||||
@@ -420,74 +420,3 @@ def write_coverage(run_dir: Path, document: dict[str, Any]) -> Path:
|
|||||||
len(document.get("gaps", [])),
|
len(document.get("gaps", [])),
|
||||||
)
|
)
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
def render_coverage_markdown(document: dict[str, Any]) -> str:
|
|
||||||
"""Render the coverage appendix appended to the executive report.
|
|
||||||
|
|
||||||
Built from the document rather than written by an agent, so the table in
|
|
||||||
the client-facing report cannot drift from the ledger it summarizes.
|
|
||||||
"""
|
|
||||||
summary = document.get("summary", {})
|
|
||||||
completeness = document.get("completeness", {})
|
|
||||||
entries = document.get("entries", [])
|
|
||||||
gaps = document.get("gaps", [])
|
|
||||||
|
|
||||||
lines = [
|
|
||||||
"# Coverage",
|
|
||||||
"",
|
|
||||||
"Surfaces assessed during this engagement and how each was closed. Entries are "
|
|
||||||
"reported by the testing agents; the completeness notes below are recorded by "
|
|
||||||
"the runtime.",
|
|
||||||
"",
|
|
||||||
f"**Surfaces reviewed:** {summary.get('surfaces_reviewed', 0)} ",
|
|
||||||
f"**Findings filed:** {summary.get('findings_filed', 0)} ",
|
|
||||||
f"**Open items and gaps:** {summary.get('gaps', 0)}",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
|
|
||||||
if entries:
|
|
||||||
lines += [
|
|
||||||
"| Surface | Risk Area | Outcome | Evidence |",
|
|
||||||
"| --- | --- | --- | --- |",
|
|
||||||
]
|
|
||||||
lines += [
|
|
||||||
"| {} | {} | {} | {} |".format(
|
|
||||||
_cell(entry.get("surface")),
|
|
||||||
_cell(entry.get("risk_area")),
|
|
||||||
_cell(entry.get("outcome_label") or entry.get("outcome")),
|
|
||||||
_cell(entry.get("evidence")),
|
|
||||||
)
|
|
||||||
for entry in entries
|
|
||||||
]
|
|
||||||
lines.append("")
|
|
||||||
else:
|
|
||||||
lines += [
|
|
||||||
"No surfaces were recorded for this scan. The absence of findings below "
|
|
||||||
"cannot be read as evidence that any particular area was tested.",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
|
|
||||||
if gaps:
|
|
||||||
lines += ["## Not Covered", ""]
|
|
||||||
lines += [f"- {_inline(gap.get('detail'))}" for gap in gaps]
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
caveats = completeness.get("caveats") or []
|
|
||||||
if caveats:
|
|
||||||
lines += ["## Completeness", ""]
|
|
||||||
lines += [f"- {_inline(caveat)}" for caveat in caveats]
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
|
|
||||||
def _cell(value: Any) -> str:
|
|
||||||
"""Flatten *value* for a markdown table cell."""
|
|
||||||
text = _inline(value)
|
|
||||||
return text.replace("|", "\\|") if text else "—"
|
|
||||||
|
|
||||||
|
|
||||||
def _inline(value: Any) -> str:
|
|
||||||
"""Collapse *value* to a single line of plain text."""
|
|
||||||
return " ".join(str(value or "").split())
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from agents.usage import Usage
|
|||||||
from strix.config import codex
|
from strix.config import codex
|
||||||
from strix.config.loader import load_settings
|
from strix.config.loader import load_settings
|
||||||
from strix.core.paths import run_dir_for, runtime_state_dir
|
from strix.core.paths import run_dir_for, runtime_state_dir
|
||||||
from strix.report.coverage import render_coverage_markdown, write_coverage
|
from strix.report.coverage import write_coverage
|
||||||
from strix.report.sarif import write_sarif
|
from strix.report.sarif import write_sarif
|
||||||
from strix.report.usage import LLMUsageLedger
|
from strix.report.usage import LLMUsageLedger
|
||||||
from strix.report.writer import (
|
from strix.report.writer import (
|
||||||
@@ -469,11 +469,7 @@ class ReportState:
|
|||||||
logger.exception("coverage.json write failed (non-fatal)")
|
logger.exception("coverage.json write failed (non-fatal)")
|
||||||
|
|
||||||
if self.final_scan_result:
|
if self.final_scan_result:
|
||||||
write_executive_report(
|
write_executive_report(run_dir, self.final_scan_result)
|
||||||
run_dir,
|
|
||||||
self.final_scan_result,
|
|
||||||
render_coverage_markdown(coverage) if coverage else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.vulnerability_reports:
|
if self.vulnerability_reports:
|
||||||
write_vulnerabilities(run_dir, self.vulnerability_reports, self._saved_vuln_ids)
|
write_vulnerabilities(run_dir, self.vulnerability_reports, self._saved_vuln_ids)
|
||||||
|
|||||||
+5
-13
@@ -113,20 +113,12 @@ def write_run_record(run_dir: Path, run_record: dict[str, Any]) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def write_executive_report(
|
def write_executive_report(run_dir: Path, final_scan_result: str) -> None:
|
||||||
run_dir: Path,
|
|
||||||
final_scan_result: str,
|
|
||||||
coverage_markdown: str | None = None,
|
|
||||||
) -> None:
|
|
||||||
path = run_dir / "penetration_test_report.md"
|
path = run_dir / "penetration_test_report.md"
|
||||||
sections = [
|
with path.open("w", encoding="utf-8") as f:
|
||||||
"# Security Penetration Test Report\n",
|
f.write("# Security Penetration Test Report\n\n")
|
||||||
f"**Generated:** {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S UTC')}\n",
|
f.write(f"**Generated:** {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S UTC')}\n\n")
|
||||||
f"{final_scan_result}\n",
|
f.write(f"{final_scan_result}\n")
|
||||||
]
|
|
||||||
if coverage_markdown:
|
|
||||||
sections.append(f"{coverage_markdown.rstrip()}\n")
|
|
||||||
atomic_write_text(path, "\n".join(sections))
|
|
||||||
logger.info("Saved final penetration test report to: %s", path)
|
logger.info("Saved final penetration test report to: %s", path)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -229,13 +229,7 @@ async def finish_scan(
|
|||||||
- ``methodology`` — frameworks followed (OWASP WSTG, PTES,
|
- ``methodology`` — frameworks followed (OWASP WSTG, PTES,
|
||||||
OSSTMM, NIST), engagement type (black/gray/white box), scope
|
OSSTMM, NIST), engagement type (black/gray/white box), scope
|
||||||
and constraints, categories of testing performed. **No**
|
and constraints, categories of testing performed. **No**
|
||||||
internal execution detail. Do **not** hand-write a table of
|
internal execution detail.
|
||||||
reviewed surfaces here: the report gains a ``Coverage``
|
|
||||||
section rendered directly from the coverage ledger, and a
|
|
||||||
transcribed copy would only drift from it. Describe the
|
|
||||||
approach and the classes of testing, and reconcile the ledger
|
|
||||||
via ``list_coverage`` (step 5) so that rendered section is
|
|
||||||
complete and honest.
|
|
||||||
- ``technical_analysis`` — consolidated findings overview with
|
- ``technical_analysis`` — consolidated findings overview with
|
||||||
severity model and systemic root causes. Reference individual
|
severity model and systemic root causes. Reference individual
|
||||||
vuln reports for repro steps; don't duplicate raw evidence.
|
vuln reports for repro steps; don't duplicate raw evidence.
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ from strix.report.coverage import (
|
|||||||
_SKILL_PHRASINGS,
|
_SKILL_PHRASINGS,
|
||||||
build_coverage_document,
|
build_coverage_document,
|
||||||
read_agent_graph,
|
read_agent_graph,
|
||||||
render_coverage_markdown,
|
|
||||||
write_coverage,
|
write_coverage,
|
||||||
)
|
)
|
||||||
from strix.skills import get_available_skills
|
from strix.skills import get_available_skills
|
||||||
@@ -171,27 +170,6 @@ def test_write_coverage_emits_a_top_level_artifact(tmp_path: Path) -> None:
|
|||||||
assert json.loads(path.read_text(encoding="utf-8"))["schema_version"] == 1
|
assert json.loads(path.read_text(encoding="utf-8"))["schema_version"] == 1
|
||||||
|
|
||||||
|
|
||||||
def test_markdown_renders_a_surface_table() -> None:
|
|
||||||
markdown = render_coverage_markdown(_document())
|
|
||||||
|
|
||||||
assert "# Coverage" in markdown
|
|
||||||
assert "| POST /api/orders/{id} | object-level authorization | No issue identified |" in (
|
|
||||||
markdown
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_markdown_says_so_when_nothing_was_recorded() -> None:
|
|
||||||
markdown = render_coverage_markdown(_document(entries=[], agent_graph={}))
|
|
||||||
|
|
||||||
assert "cannot be read as evidence" in markdown
|
|
||||||
|
|
||||||
|
|
||||||
def test_markdown_cell_escapes_pipes() -> None:
|
|
||||||
markdown = render_coverage_markdown(_document(entries=[_entry(surface="a|b")]))
|
|
||||||
|
|
||||||
assert "| a\\|b |" in markdown
|
|
||||||
|
|
||||||
|
|
||||||
def test_read_agent_graph_tolerates_a_missing_or_corrupt_snapshot(tmp_path: Path) -> None:
|
def test_read_agent_graph_tolerates_a_missing_or_corrupt_snapshot(tmp_path: Path) -> None:
|
||||||
assert read_agent_graph(tmp_path) == {}
|
assert read_agent_graph(tmp_path) == {}
|
||||||
|
|
||||||
|
|||||||
@@ -206,11 +206,3 @@ def test_render_vulnerability_md_surfaces_calibration_metadata() -> None:
|
|||||||
assert "## Confidence Rationale" in md
|
assert "## Confidence Rationale" in md
|
||||||
assert "## What Would Change This Severity" in md
|
assert "## What Would Change This Severity" in md
|
||||||
assert "## Fix Verification" in md
|
assert "## Fix Verification" in md
|
||||||
|
|
||||||
|
|
||||||
def test_write_executive_report_appends_the_coverage_section(tmp_path: Path) -> None:
|
|
||||||
write_executive_report(tmp_path, "Scan complete.", "# Coverage\n\nNothing tested.\n")
|
|
||||||
content = (tmp_path / "penetration_test_report.md").read_text(encoding="utf-8")
|
|
||||||
|
|
||||||
assert content.index("Scan complete.") < content.index("# Coverage")
|
|
||||||
assert "Nothing tested." in content
|
|
||||||
|
|||||||
@@ -45,17 +45,6 @@ def test_coverage_is_written_beside_the_other_artifacts(state: ReportState) -> N
|
|||||||
assert document["summary"]["surfaces_reviewed"] == 1
|
assert document["summary"]["surfaces_reviewed"] == 1
|
||||||
|
|
||||||
|
|
||||||
def test_report_carries_a_coverage_section_from_the_ledger(state: ReportState) -> None:
|
|
||||||
_record_a_cleared_surface()
|
|
||||||
state.final_scan_result = "Scan complete."
|
|
||||||
|
|
||||||
state._save_artifacts()
|
|
||||||
|
|
||||||
report = (state.get_run_dir() / "penetration_test_report.md").read_text(encoding="utf-8")
|
|
||||||
assert "# Coverage" in report
|
|
||||||
assert "POST /api/orders/{id}" in report
|
|
||||||
|
|
||||||
|
|
||||||
def test_cleared_surfaces_reach_sarif(state: ReportState) -> None:
|
def test_cleared_surfaces_reach_sarif(state: ReportState) -> None:
|
||||||
_record_a_cleared_surface()
|
_record_a_cleared_surface()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user