mirror of
https://github.com/usestrix/strix.git
synced 2026-08-18 01:39:19 +02:00
Alex Schapiro
d91ac851cf
Promote the coverage ledger from runtime state to a deliverable: coverage.json beside vulnerabilities.json, a Coverage section rendered into the report from the ledger rather than transcribed by an agent, and SARIF pass / notApplicable / open results so a consumer can tell 'tested and clean' from 'never tested'. Ground it in what the runtime observed rather than only what agents claimed: a risk class an agent carried a skill for and never accounted for is published as a gap (and surfaced back to the root agent from finish_scan while it can still act), and a run cut short is stamped incomplete on both the artifact and the SARIF invocation. Also: make the ledger's duplicate check and insertion one critical section and persist under the lock; key a checkout and the URL it was cloned from onto one threat-model identity; render the calibration metadata (counterevidence, confidence, severity change conditions, fix verification) that was being stored and then dropped.
204 lines
6.6 KiB
Python
204 lines
6.6 KiB
Python
"""Tests for the coverage artifact assembled in strix.report.coverage."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from strix.report.coverage import (
|
|
build_coverage_document,
|
|
read_agent_graph,
|
|
render_coverage_markdown,
|
|
write_coverage,
|
|
)
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
|
|
def _entry(**overrides: Any) -> dict[str, Any]:
|
|
base: dict[str, Any] = {
|
|
"surface": "POST /api/orders/{id}",
|
|
"risk_area": "object-level authorization",
|
|
"outcome": "no_issue_found",
|
|
"evidence": "Two tenants tested; both received 403.",
|
|
"agent_id": "agent-1",
|
|
"agent_name": "authz-tester",
|
|
"created_at": "2026-07-02 10:00:00 UTC",
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _graph(**overrides: Any) -> dict[str, Any]:
|
|
base: dict[str, Any] = {
|
|
"statuses": {"agent-1": "completed"},
|
|
"names": {"agent-1": "authz-tester"},
|
|
"metadata": {"agent-1": {"skills": ["idor"], "task": "authz review"}},
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _document(**overrides: Any) -> dict[str, Any]:
|
|
kwargs: dict[str, Any] = {
|
|
"run_record": {"run_id": "r1", "run_name": "run-1", "status": "completed"},
|
|
"entries": [_entry()],
|
|
"agent_graph": _graph(),
|
|
"vulnerability_reports": [],
|
|
}
|
|
kwargs.update(overrides)
|
|
return build_coverage_document(**kwargs)
|
|
|
|
|
|
def test_document_reports_surfaces_and_outcomes() -> None:
|
|
doc = _document()
|
|
|
|
assert doc["summary"]["surfaces_reviewed"] == 1
|
|
assert doc["summary"]["outcomes"] == {"no_issue_found": 1}
|
|
assert doc["entries"][0]["outcome_label"] == "No issue identified"
|
|
assert doc["entries"][0]["recorded_by"] == "authz-tester"
|
|
|
|
|
|
def test_ledger_entries_are_labelled_as_agent_reported() -> None:
|
|
"""A reader has to be able to tell a self-report from an observation."""
|
|
doc = _document()
|
|
|
|
assert doc["entries"][0]["source"] == "agent_reported"
|
|
assert doc["machine_observed"]["source"] == "runtime"
|
|
assert doc["machine_observed"]["skills_exercised"] == ["idor"]
|
|
|
|
|
|
def test_assigned_risk_skill_without_coverage_becomes_a_gap() -> None:
|
|
"""An agent carrying the sql_injection skill that records nothing about it
|
|
leaves the class unexamined, not clean."""
|
|
doc = _document(
|
|
agent_graph=_graph(
|
|
metadata={"agent-1": {"skills": ["idor", "sql_injection"], "task": "review"}}
|
|
)
|
|
)
|
|
|
|
gaps = [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
|
|
assert [gap["risk_area"] for gap in gaps] == ["sql injection"]
|
|
|
|
|
|
def test_recorded_risk_class_is_not_reported_as_a_gap() -> None:
|
|
doc = _document(
|
|
entries=[_entry(risk_area="SQL injection", surface="GET /search?q=")],
|
|
agent_graph=_graph(metadata={"agent-1": {"skills": ["sql_injection"]}}),
|
|
)
|
|
|
|
assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
|
|
|
|
|
|
def test_synonym_phrasing_counts_as_recorded_coverage() -> None:
|
|
"""The ledger says "object-level authorization"; the skill is called idor."""
|
|
doc = _document(agent_graph=_graph(metadata={"agent-1": {"skills": ["idor"]}}))
|
|
|
|
assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
|
|
|
|
|
|
def test_non_risk_skills_carry_no_coverage_obligation() -> None:
|
|
"""Tooling skills describe how an agent works, not what it hunts."""
|
|
doc = _document(agent_graph=_graph(metadata={"agent-1": {"skills": ["idor", "caido"]}}))
|
|
|
|
assert not [gap for gap in doc["gaps"] if gap.get("risk_area") == "caido"]
|
|
|
|
|
|
def test_agent_that_recorded_nothing_is_a_gap() -> None:
|
|
doc = _document(
|
|
agent_graph=_graph(
|
|
statuses={"agent-1": "completed", "agent-2": "completed"},
|
|
names={"agent-1": "authz-tester", "agent-2": "recon"},
|
|
metadata={},
|
|
)
|
|
)
|
|
|
|
silent = [gap for gap in doc["gaps"] if gap["kind"] == "agent_recorded_no_coverage"]
|
|
assert [gap["agent_name"] for gap in silent] == ["recon"]
|
|
|
|
|
|
def test_needs_follow_up_is_carried_as_an_open_gap() -> None:
|
|
doc = _document(
|
|
entries=[_entry(outcome="needs_follow_up", evidence="Auth wall blocked testing.")]
|
|
)
|
|
|
|
assert doc["gaps"][0]["kind"] == "needs_follow_up"
|
|
assert doc["gaps"][0]["detail"] == "Auth wall blocked testing."
|
|
|
|
|
|
def test_completed_run_with_finished_agents_is_complete() -> None:
|
|
doc = _document(exit_reason="finished_by_tool")
|
|
|
|
assert doc["completeness"]["complete"] is True
|
|
assert doc["completeness"]["caveats"] == []
|
|
|
|
|
|
def test_budget_exhausted_run_is_not_a_complete_record() -> None:
|
|
"""A truncated scan must not read like a clean one."""
|
|
doc = _document(exit_reason="budget_exhausted")
|
|
|
|
assert doc["completeness"]["complete"] is False
|
|
assert "budget_exhausted" in doc["completeness"]["caveats"][0]
|
|
|
|
|
|
def test_unfinished_agent_makes_the_record_partial() -> None:
|
|
doc = _document(
|
|
agent_graph=_graph(statuses={"agent-1": "crashed"}),
|
|
exit_reason="finished_by_tool",
|
|
)
|
|
|
|
assert doc["completeness"]["complete"] is False
|
|
assert "authz-tester" in doc["completeness"]["caveats"][0]
|
|
|
|
|
|
def test_failed_run_status_makes_the_record_partial() -> None:
|
|
doc = _document(
|
|
run_record={"run_id": "r1", "status": "failed"},
|
|
exit_reason="finished_by_tool",
|
|
)
|
|
|
|
assert doc["completeness"]["complete"] is False
|
|
|
|
|
|
def test_write_coverage_emits_a_top_level_artifact(tmp_path: Path) -> None:
|
|
path = write_coverage(tmp_path, _document())
|
|
|
|
assert path == tmp_path / "coverage.json"
|
|
assert json.loads(path.read_text(encoding="utf-8"))["schema_version"] == 1
|
|
|
|
|
|
def test_markdown_renders_a_surface_table() -> None:
|
|
markdown = render_coverage_markdown(_document())
|
|
|
|
assert "# Coverage" in markdown
|
|
assert "| POST /api/orders/{id} | object-level authorization | No issue identified |" in (
|
|
markdown
|
|
)
|
|
|
|
|
|
def test_markdown_says_so_when_nothing_was_recorded() -> None:
|
|
markdown = render_coverage_markdown(_document(entries=[], agent_graph={}))
|
|
|
|
assert "cannot be read as evidence" in markdown
|
|
|
|
|
|
def test_markdown_cell_escapes_pipes() -> None:
|
|
markdown = render_coverage_markdown(_document(entries=[_entry(surface="a|b")]))
|
|
|
|
assert "| a\\|b |" in markdown
|
|
|
|
|
|
def test_read_agent_graph_tolerates_a_missing_or_corrupt_snapshot(tmp_path: Path) -> None:
|
|
assert read_agent_graph(tmp_path) == {}
|
|
|
|
(tmp_path / "agents.json").write_text("{not json", encoding="utf-8")
|
|
assert read_agent_graph(tmp_path) == {}
|
|
|
|
|
|
def test_read_agent_graph_loads_a_snapshot(tmp_path: Path) -> None:
|
|
(tmp_path / "agents.json").write_text(json.dumps(_graph()), encoding="utf-8")
|
|
|
|
assert read_agent_graph(tmp_path)["names"] == {"agent-1": "authz-tester"}
|