mirror of
https://github.com/usestrix/strix.git
synced 2026-08-19 18:13:34 +02:00
Alex Schapiro
d91ac851cf
Promote the coverage ledger from runtime state to a deliverable: coverage.json beside vulnerabilities.json, a Coverage section rendered into the report from the ledger rather than transcribed by an agent, and SARIF pass / notApplicable / open results so a consumer can tell 'tested and clean' from 'never tested'. Ground it in what the runtime observed rather than only what agents claimed: a risk class an agent carried a skill for and never accounted for is published as a gap (and surfaced back to the root agent from finish_scan while it can still act), and a run cut short is stamped incomplete on both the artifact and the SARIF invocation. Also: make the ledger's duplicate check and insertion one critical section and persist under the lock; key a checkout and the URL it was cloned from onto one threat-model identity; render the calibration metadata (counterevidence, confidence, severity change conditions, fix verification) that was being stored and then dropped.
78 lines
2.4 KiB
Python
78 lines
2.4 KiB
Python
"""coverage.json is a deliverable artifact, not runtime state."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from strix.core.paths import runtime_state_dir
|
|
from strix.report.state import ReportState
|
|
from strix.tools.coverage.tools import _record_impl, hydrate_coverage_from_disk
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
|
|
@pytest.fixture
|
|
def state(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> ReportState:
|
|
monkeypatch.chdir(tmp_path)
|
|
report_state = ReportState(run_name="run-1")
|
|
hydrate_coverage_from_disk(runtime_state_dir(report_state.get_run_dir()))
|
|
return report_state
|
|
|
|
|
|
def _record_a_cleared_surface() -> None:
|
|
_record_impl(
|
|
surface="POST /api/orders/{id}",
|
|
risk_area="SQL injection",
|
|
outcome="no_issue_found",
|
|
evidence="14 parameters fuzzed; every query parameterized.",
|
|
agent_id="agent-1",
|
|
agent_name="injection-tester",
|
|
)
|
|
|
|
|
|
def test_coverage_is_written_beside_the_other_artifacts(state: ReportState) -> None:
|
|
_record_a_cleared_surface()
|
|
|
|
state._save_artifacts()
|
|
|
|
document = json.loads((state.get_run_dir() / "coverage.json").read_text(encoding="utf-8"))
|
|
assert document["entries"][0]["risk_area"] == "SQL injection"
|
|
assert document["summary"]["surfaces_reviewed"] == 1
|
|
|
|
|
|
def test_report_carries_a_coverage_section_from_the_ledger(state: ReportState) -> None:
|
|
_record_a_cleared_surface()
|
|
state.final_scan_result = "Scan complete."
|
|
|
|
state._save_artifacts()
|
|
|
|
report = (state.get_run_dir() / "penetration_test_report.md").read_text(encoding="utf-8")
|
|
assert "# Coverage" in report
|
|
assert "POST /api/orders/{id}" in report
|
|
|
|
|
|
def test_cleared_surfaces_reach_sarif(state: ReportState) -> None:
|
|
_record_a_cleared_surface()
|
|
|
|
state._save_artifacts()
|
|
|
|
sarif = json.loads((state.get_run_dir() / "findings.sarif").read_text(encoding="utf-8"))
|
|
results = sarif["runs"][0]["results"]
|
|
assert [result["kind"] for result in results] == ["pass"]
|
|
|
|
|
|
def test_artifacts_still_land_when_coverage_is_empty(state: ReportState) -> None:
|
|
state.final_scan_result = "Scan complete."
|
|
|
|
state._save_artifacts()
|
|
|
|
run_dir = state.get_run_dir()
|
|
assert (run_dir / "penetration_test_report.md").is_file()
|
|
document = json.loads((run_dir / "coverage.json").read_text(encoding="utf-8"))
|
|
assert document["entries"] == []
|