coverage: make the negative space of a scan a first-class artifact

Promote the coverage ledger from runtime state to a deliverable:
coverage.json beside vulnerabilities.json, a Coverage section rendered
into the report from the ledger rather than transcribed by an agent, and
SARIF pass / notApplicable / open results so a consumer can tell 'tested
and clean' from 'never tested'.

Ground it in what the runtime observed rather than only what agents
claimed: a risk class an agent carried a skill for and never accounted
for is published as a gap (and surfaced back to the root agent from
finish_scan while it can still act), and a run cut short is stamped
incomplete on both the artifact and the SARIF invocation.

Also: make the ledger's duplicate check and insertion one critical
section and persist under the lock; key a checkout and the URL it was
cloned from onto one threat-model identity; render the calibration
metadata (counterevidence, confidence, severity change conditions, fix
verification) that was being stored and then dropped.

tests/test_coverage_tool.py

@@ -2,6 +2,9 @@
 
 from __future__ import annotations
 
+import json
+import threading
+from concurrent.futures import ThreadPoolExecutor
 from typing import TYPE_CHECKING, Any
 
 import pytest
@@ -237,3 +240,45 @@ def test_a_different_risk_area_on_one_surface_is_still_its_own_entry() -> None:
 
     assert second["success"] is True
     assert len(get_coverage_entries()) == 2
+
+
+def test_concurrent_records_of_one_surface_yield_a_single_row() -> None:
+    """Duplicate detection and insertion must be one critical section.
+
+    Two agents recording the same surface at the same moment would otherwise
+    both pass the "no duplicate" check, and the report would show a stale
+    conclusion beside its replacement — the exact outcome the rejection exists
+    to prevent.
+    """
+    barrier = threading.Barrier(8)
+
+    def attempt(index: int) -> dict[str, Any]:
+        barrier.wait()
+        return _record(agent_id=f"agent-{index}", agent_name=f"tester-{index}")
+
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        results = list(pool.map(attempt, range(8)))
+
+    assert sum(1 for result in results if result["success"]) == 1
+    assert len(get_coverage_entries()) == 1
+
+
+def test_concurrent_records_all_survive_persistence(coverage_store: Path) -> None:
+    """A writer holding an older snapshot must not win the rename.
+
+    If it did, the mirror would come back short on resume and coverage
+    recorded before a crash would silently disappear from the report.
+    """
+    barrier = threading.Barrier(8)
+
+    def attempt(index: int) -> dict[str, Any]:
+        barrier.wait()
+        return _record(surface=f"GET /api/resource/{index}", agent_id=f"agent-{index}")
+
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        list(pool.map(attempt, range(8)))
+
+    persisted = json.loads((coverage_store / "coverage.json").read_text(encoding="utf-8"))
+    assert len(persisted) == 8
+    hydrate_coverage_from_disk(coverage_store)
+    assert len(get_coverage_entries()) == 8

tests/test_finish_coverage_gate.py

@@ -0,0 +1,65 @@
+"""finish_scan confronts the root agent with the coverage the runtime can see."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from strix.tools.coverage.tools import _record_impl, hydrate_coverage_from_disk
+from strix.tools.finish.tool import _coverage_summary
+
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+_GRAPH = {
+    "statuses": {"agent-1": "completed"},
+    "names": {"agent-1": "injection-tester"},
+    "metadata": {"agent-1": {"skills": ["sql_injection", "xss"]}},
+}
+
+
+@pytest.fixture(autouse=True)
+def _empty_ledger(tmp_path: Path) -> None:
+    hydrate_coverage_from_disk(tmp_path)
+
+
+def _record(risk_area: str) -> None:
+    _record_impl(
+        surface="POST /api/orders/{id}",
+        risk_area=risk_area,
+        outcome="no_issue_found",
+        evidence="Parameters fuzzed; no anomalies.",
+        agent_id="agent-1",
+        agent_name="injection-tester",
+    )
+
+
+def test_unrecorded_risk_class_is_reported_back_to_the_root_agent() -> None:
+    _record("SQL injection")
+
+    summary = _coverage_summary(_GRAPH)
+
+    assert summary["coverage_recorded"] == 1
+    assert len(summary["coverage_gaps"]) == 1
+    assert "xss" in summary["coverage_gaps"][0]
+    assert "unexamined" in summary["coverage_gap_warning"]
+
+
+def test_fully_accounted_coverage_raises_no_gap_warning() -> None:
+    _record("SQL injection")
+    _record("cross-site scripting")
+
+    summary = _coverage_summary(_GRAPH)
+
+    assert "coverage_gaps" not in summary
+    assert "coverage_gap_warning" not in summary
+
+
+def test_an_empty_ledger_still_warns_first() -> None:
+    summary = _coverage_summary(_GRAPH)
+
+    assert summary["coverage_recorded"] == 0
+    assert "No coverage was recorded" in summary["coverage_warning"]

tests/test_report_coverage.py

@@ -0,0 +1,203 @@
+"""Tests for the coverage artifact assembled in strix.report.coverage."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+from strix.report.coverage import (
+    build_coverage_document,
+    read_agent_graph,
+    render_coverage_markdown,
+    write_coverage,
+)
+
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def _entry(**overrides: Any) -> dict[str, Any]:
+    base: dict[str, Any] = {
+        "surface": "POST /api/orders/{id}",
+        "risk_area": "object-level authorization",
+        "outcome": "no_issue_found",
+        "evidence": "Two tenants tested; both received 403.",
+        "agent_id": "agent-1",
+        "agent_name": "authz-tester",
+        "created_at": "2026-07-02 10:00:00 UTC",
+    }
+    base.update(overrides)
+    return base
+
+
+def _graph(**overrides: Any) -> dict[str, Any]:
+    base: dict[str, Any] = {
+        "statuses": {"agent-1": "completed"},
+        "names": {"agent-1": "authz-tester"},
+        "metadata": {"agent-1": {"skills": ["idor"], "task": "authz review"}},
+    }
+    base.update(overrides)
+    return base
+
+
+def _document(**overrides: Any) -> dict[str, Any]:
+    kwargs: dict[str, Any] = {
+        "run_record": {"run_id": "r1", "run_name": "run-1", "status": "completed"},
+        "entries": [_entry()],
+        "agent_graph": _graph(),
+        "vulnerability_reports": [],
+    }
+    kwargs.update(overrides)
+    return build_coverage_document(**kwargs)
+
+
+def test_document_reports_surfaces_and_outcomes() -> None:
+    doc = _document()
+
+    assert doc["summary"]["surfaces_reviewed"] == 1
+    assert doc["summary"]["outcomes"] == {"no_issue_found": 1}
+    assert doc["entries"][0]["outcome_label"] == "No issue identified"
+    assert doc["entries"][0]["recorded_by"] == "authz-tester"
+
+
+def test_ledger_entries_are_labelled_as_agent_reported() -> None:
+    """A reader has to be able to tell a self-report from an observation."""
+    doc = _document()
+
+    assert doc["entries"][0]["source"] == "agent_reported"
+    assert doc["machine_observed"]["source"] == "runtime"
+    assert doc["machine_observed"]["skills_exercised"] == ["idor"]
+
+
+def test_assigned_risk_skill_without_coverage_becomes_a_gap() -> None:
+    """An agent carrying the sql_injection skill that records nothing about it
+    leaves the class unexamined, not clean."""
+    doc = _document(
+        agent_graph=_graph(
+            metadata={"agent-1": {"skills": ["idor", "sql_injection"], "task": "review"}}
+        )
+    )
+
+    gaps = [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
+    assert [gap["risk_area"] for gap in gaps] == ["sql injection"]
+
+
+def test_recorded_risk_class_is_not_reported_as_a_gap() -> None:
+    doc = _document(
+        entries=[_entry(risk_area="SQL injection", surface="GET /search?q=")],
+        agent_graph=_graph(metadata={"agent-1": {"skills": ["sql_injection"]}}),
+    )
+
+    assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
+
+
+def test_synonym_phrasing_counts_as_recorded_coverage() -> None:
+    """The ledger says "object-level authorization"; the skill is called idor."""
+    doc = _document(agent_graph=_graph(metadata={"agent-1": {"skills": ["idor"]}}))
+
+    assert not [gap for gap in doc["gaps"] if gap["kind"] == "unrecorded_risk_class"]
+
+
+def test_non_risk_skills_carry_no_coverage_obligation() -> None:
+    """Tooling skills describe how an agent works, not what it hunts."""
+    doc = _document(agent_graph=_graph(metadata={"agent-1": {"skills": ["idor", "caido"]}}))
+
+    assert not [gap for gap in doc["gaps"] if gap.get("risk_area") == "caido"]
+
+
+def test_agent_that_recorded_nothing_is_a_gap() -> None:
+    doc = _document(
+        agent_graph=_graph(
+            statuses={"agent-1": "completed", "agent-2": "completed"},
+            names={"agent-1": "authz-tester", "agent-2": "recon"},
+            metadata={},
+        )
+    )
+
+    silent = [gap for gap in doc["gaps"] if gap["kind"] == "agent_recorded_no_coverage"]
+    assert [gap["agent_name"] for gap in silent] == ["recon"]
+
+
+def test_needs_follow_up_is_carried_as_an_open_gap() -> None:
+    doc = _document(
+        entries=[_entry(outcome="needs_follow_up", evidence="Auth wall blocked testing.")]
+    )
+
+    assert doc["gaps"][0]["kind"] == "needs_follow_up"
+    assert doc["gaps"][0]["detail"] == "Auth wall blocked testing."
+
+
+def test_completed_run_with_finished_agents_is_complete() -> None:
+    doc = _document(exit_reason="finished_by_tool")
+
+    assert doc["completeness"]["complete"] is True
+    assert doc["completeness"]["caveats"] == []
+
+
+def test_budget_exhausted_run_is_not_a_complete_record() -> None:
+    """A truncated scan must not read like a clean one."""
+    doc = _document(exit_reason="budget_exhausted")
+
+    assert doc["completeness"]["complete"] is False
+    assert "budget_exhausted" in doc["completeness"]["caveats"][0]
+
+
+def test_unfinished_agent_makes_the_record_partial() -> None:
+    doc = _document(
+        agent_graph=_graph(statuses={"agent-1": "crashed"}),
+        exit_reason="finished_by_tool",
+    )
+
+    assert doc["completeness"]["complete"] is False
+    assert "authz-tester" in doc["completeness"]["caveats"][0]
+
+
+def test_failed_run_status_makes_the_record_partial() -> None:
+    doc = _document(
+        run_record={"run_id": "r1", "status": "failed"},
+        exit_reason="finished_by_tool",
+    )
+
+    assert doc["completeness"]["complete"] is False
+
+
+def test_write_coverage_emits_a_top_level_artifact(tmp_path: Path) -> None:
+    path = write_coverage(tmp_path, _document())
+
+    assert path == tmp_path / "coverage.json"
+    assert json.loads(path.read_text(encoding="utf-8"))["schema_version"] == 1
+
+
+def test_markdown_renders_a_surface_table() -> None:
+    markdown = render_coverage_markdown(_document())
+
+    assert "# Coverage" in markdown
+    assert "| POST /api/orders/{id} | object-level authorization | No issue identified |" in (
+        markdown
+    )
+
+
+def test_markdown_says_so_when_nothing_was_recorded() -> None:
+    markdown = render_coverage_markdown(_document(entries=[], agent_graph={}))
+
+    assert "cannot be read as evidence" in markdown
+
+
+def test_markdown_cell_escapes_pipes() -> None:
+    markdown = render_coverage_markdown(_document(entries=[_entry(surface="a|b")]))
+
+    assert "| a\\|b |" in markdown
+
+
+def test_read_agent_graph_tolerates_a_missing_or_corrupt_snapshot(tmp_path: Path) -> None:
+    assert read_agent_graph(tmp_path) == {}
+
+    (tmp_path / "agents.json").write_text("{not json", encoding="utf-8")
+    assert read_agent_graph(tmp_path) == {}
+
+
+def test_read_agent_graph_loads_a_snapshot(tmp_path: Path) -> None:
+    (tmp_path / "agents.json").write_text(json.dumps(_graph()), encoding="utf-8")
+
+    assert read_agent_graph(tmp_path)["names"] == {"agent-1": "authz-tester"}

tests/test_report_writer.py

@@ -179,3 +179,38 @@ def test_write_executive_report_writes_markdown(tmp_path: Path) -> None:
     content = (tmp_path / "penetration_test_report.md").read_text(encoding="utf-8")
     assert "# Security Penetration Test Report" in content
     assert "Scan complete. No critical issues." in content
+
+
+def test_render_vulnerability_md_surfaces_calibration_metadata() -> None:
+    """Confidence, the case against the finding, and retest status are part of
+    the deliverable — storing them without rendering hides the reasoning."""
+    md = render_vulnerability_md(
+        {
+            "id": "vuln-0009",
+            "title": "SSRF in URL preview",
+            "severity": "high",
+            "timestamp": "2026-07-02 10:00:00 UTC",
+            "description": "Fetches user-supplied URLs.",
+            "confidence": "medium",
+            "counterevidence": "Egress appears filtered at the network layer.",
+            "confidence_rationale": "Reproduced once out of three attempts.",
+            "severity_change_conditions": "Critical if egress filtering is removed.",
+            "remediation_steps": "Allowlist destinations.",
+            "fix_verification": "Not retested.",
+        }
+    )
+
+    assert "**Confidence:** Medium" in md
+    assert "## Counterevidence" in md
+    assert "Egress appears filtered at the network layer." in md
+    assert "## Confidence Rationale" in md
+    assert "## What Would Change This Severity" in md
+    assert "## Fix Verification" in md
+
+
+def test_write_executive_report_appends_the_coverage_section(tmp_path: Path) -> None:
+    write_executive_report(tmp_path, "Scan complete.", "# Coverage\n\nNothing tested.\n")
+    content = (tmp_path / "penetration_test_report.md").read_text(encoding="utf-8")
+
+    assert content.index("Scan complete.") < content.index("# Coverage")
+    assert "Nothing tested." in content

tests/test_sarif.py

@@ -242,3 +242,132 @@ def test_write_sarif_replaces_atomically_no_partial_on_reemit(tmp_path: Path) ->
     assert leftovers == []
     # And it parses as a complete document with both findings.
     assert len(_read(tmp_path)["runs"][0]["results"]) == 2
+
+
+def _coverage(*entries: dict[str, Any], **overrides: Any) -> dict[str, Any]:
+    doc: dict[str, Any] = {
+        "entries": list(entries),
+        "completeness": {"complete": True, "caveats": []},
+    }
+    doc.update(overrides)
+    return doc
+
+
+def _coverage_entry(**overrides: Any) -> dict[str, Any]:
+    base: dict[str, Any] = {
+        "surface": "POST /api/orders/{id}",
+        "risk_area": "SQL injection",
+        "outcome": "no_issue_found",
+        "outcome_label": "No issue identified",
+        "evidence": "14 parameters fuzzed; all queries parameterized.",
+        "recorded_by": "injection-tester",
+        "source": "agent_reported",
+    }
+    base.update(overrides)
+    return base
+
+
+def test_cleared_surface_becomes_a_passing_result(tmp_path: Path) -> None:
+    """ "Tested and clean" is a SARIF pass, not an absent result."""
+    write_sarif(tmp_path, [], coverage=_coverage(_coverage_entry()))
+    results = _read(tmp_path)["runs"][0]["results"]
+
+    assert len(results) == 1
+    assert results[0]["kind"] == "pass"
+    # SARIF requires level "none" on any result that is not a failure.
+    assert results[0]["level"] == "none"
+    assert "14 parameters fuzzed" in results[0]["message"]["text"]
+
+
+def test_coverage_outcomes_map_to_their_sarif_kinds(tmp_path: Path) -> None:
+    write_sarif(
+        tmp_path,
+        [],
+        coverage=_coverage(
+            _coverage_entry(outcome="ruled_out", risk_area="XSS"),
+            _coverage_entry(outcome="not_applicable", risk_area="XXE"),
+            _coverage_entry(outcome="needs_follow_up", risk_area="SSRF"),
+        ),
+    )
+    kinds = [result["kind"] for result in _read(tmp_path)["runs"][0]["results"]]
+
+    assert kinds == ["pass", "notApplicable", "open"]
+
+
+def test_reported_coverage_is_not_duplicated_as_a_pass(tmp_path: Path) -> None:
+    """A surface that produced a finding is already in results as a failure."""
+    write_sarif(
+        tmp_path,
+        [_finding()],
+        coverage=_coverage(_coverage_entry(outcome="reported")),
+    )
+    results = _read(tmp_path)["runs"][0]["results"]
+
+    assert len(results) == 1
+    assert results[0].get("kind", "fail") == "fail"
+
+
+def test_coverage_results_declare_their_own_rules(tmp_path: Path) -> None:
+    write_sarif(
+        tmp_path,
+        [_finding()],
+        coverage=_coverage(
+            _coverage_entry(risk_area="SQL injection"),
+            _coverage_entry(risk_area="SQL injection", surface="GET /search"),
+        ),
+    )
+    run = _read(tmp_path)["runs"][0]
+    rules = run["tool"]["driver"]["rules"]
+    coverage_rules = [rule for rule in rules if rule["id"].startswith("strix-coverage/")]
+
+    # Both entries share one rule, and every result's ruleIndex resolves to it.
+    assert len(coverage_rules) == 1
+    assert coverage_rules[0]["defaultConfiguration"]["level"] == "none"
+    for result in run["results"]:
+        assert rules[result["ruleIndex"]]["id"] == result["ruleId"]
+
+
+def test_incomplete_run_is_flagged_on_the_invocation(tmp_path: Path) -> None:
+    """A scan cut short must not be indistinguishable from a clean one."""
+    write_sarif(
+        tmp_path,
+        [],
+        coverage=_coverage(
+            _coverage_entry(),
+            completeness={"complete": False, "caveats": ["Budget exhausted."]},
+        ),
+    )
+    invocation = _read(tmp_path)["runs"][0]["invocations"][0]
+
+    assert invocation["executionSuccessful"] is False
+    assert invocation["toolExecutionNotifications"][0]["message"]["text"] == "Budget exhausted."
+
+
+def test_complete_run_reports_a_successful_invocation(tmp_path: Path) -> None:
+    write_sarif(tmp_path, [], coverage=_coverage(_coverage_entry()))
+    invocation = _read(tmp_path)["runs"][0]["invocations"][0]
+
+    assert invocation["executionSuccessful"] is True
+    assert "toolExecutionNotifications" not in invocation
+
+
+def test_calibration_metadata_survives_into_result_properties(tmp_path: Path) -> None:
+    write_sarif(
+        tmp_path,
+        [
+            _finding(
+                confidence="medium",
+                counterevidence="WAF blocks the naive payload.",
+                confidence_rationale="Reproduced once out of three attempts.",
+                severity_change_conditions="Critical if the WAF rule is removed.",
+                fix_verification="Not retested.",
+            )
+        ],
+    )
+    strix = _read(tmp_path)["runs"][0]["results"][0]["properties"]["strix"]
+
+    assert strix["confidence"] == "medium"
+    assert strix["counterevidence"] == "WAF blocks the naive payload."
+    assert strix["confidence_rationale"] == "Reproduced once out of three attempts."
+    assert strix["severity_change_conditions"] == "Critical if the WAF rule is removed."
+    assert strix["fix_verification"] == "Not retested."

tests/test_state_coverage_artifact.py

@@ -0,0 +1,77 @@
+"""coverage.json is a deliverable artifact, not runtime state."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+import pytest
+
+from strix.core.paths import runtime_state_dir
+from strix.report.state import ReportState
+from strix.tools.coverage.tools import _record_impl, hydrate_coverage_from_disk
+
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+@pytest.fixture
+def state(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> ReportState:
+    monkeypatch.chdir(tmp_path)
+    report_state = ReportState(run_name="run-1")
+    hydrate_coverage_from_disk(runtime_state_dir(report_state.get_run_dir()))
+    return report_state
+
+
+def _record_a_cleared_surface() -> None:
+    _record_impl(
+        surface="POST /api/orders/{id}",
+        risk_area="SQL injection",
+        outcome="no_issue_found",
+        evidence="14 parameters fuzzed; every query parameterized.",
+        agent_id="agent-1",
+        agent_name="injection-tester",
+    )
+
+
+def test_coverage_is_written_beside_the_other_artifacts(state: ReportState) -> None:
+    _record_a_cleared_surface()
+
+    state._save_artifacts()
+
+    document = json.loads((state.get_run_dir() / "coverage.json").read_text(encoding="utf-8"))
+    assert document["entries"][0]["risk_area"] == "SQL injection"
+    assert document["summary"]["surfaces_reviewed"] == 1
+
+
+def test_report_carries_a_coverage_section_from_the_ledger(state: ReportState) -> None:
+    _record_a_cleared_surface()
+    state.final_scan_result = "Scan complete."
+
+    state._save_artifacts()
+
+    report = (state.get_run_dir() / "penetration_test_report.md").read_text(encoding="utf-8")
+    assert "# Coverage" in report
+    assert "POST /api/orders/{id}" in report
+
+
+def test_cleared_surfaces_reach_sarif(state: ReportState) -> None:
+    _record_a_cleared_surface()
+
+    state._save_artifacts()
+
+    sarif = json.loads((state.get_run_dir() / "findings.sarif").read_text(encoding="utf-8"))
+    results = sarif["runs"][0]["results"]
+    assert [result["kind"] for result in results] == ["pass"]
+
+
+def test_artifacts_still_land_when_coverage_is_empty(state: ReportState) -> None:
+    state.final_scan_result = "Scan complete."
+
+    state._save_artifacts()
+
+    run_dir = state.get_run_dir()
+    assert (run_dir / "penetration_test_report.md").is_file()
+    document = json.loads((run_dir / "coverage.json").read_text(encoding="utf-8"))
+    assert document["entries"] == []

tests/test_threat_model_tool.py

@@ -307,3 +307,37 @@ def test_repository_subdirectory_shares_the_repository_model(tmp_path: Path) ->
     _save_impl(str(repo), _MODEL, "root")
 
     assert _get_impl(str(repo / "src"))["found"] is True
+
+
+def test_checkout_and_its_clone_url_are_one_identity(tmp_path: Path) -> None:
+    """The model an agent saves inside the checkout must be visible to an agent
+    that names the same repository by the URL it was cloned from."""
+    repo = _make_repo(tmp_path)
+    _git(repo, "remote", "add", "origin", "https://github.com/acme/billing.git")
+    _save_impl(str(repo), _MODEL, "root")
+
+    assert _get_impl("https://github.com/acme/billing")["found"] is True
+    assert _get_impl("https://github.com/acme/billing.git")["found"] is True
+
+
+def test_ssh_and_https_remotes_are_one_identity(tmp_path: Path) -> None:
+    """One repository cloned over scp-style SSH and over HTTPS is one target."""
+    over_ssh = _make_repo(tmp_path, "ssh-clone")
+    _git(over_ssh, "remote", "add", "origin", "git@github.com:acme/billing.git")
+    _save_impl(str(over_ssh), _MODEL, "root")
+
+    over_https = _make_repo(tmp_path, "https-clone")
+    _git(over_https, "remote", "add", "origin", "https://github.com/acme/billing.git")
+
+    assert _get_impl(str(over_https))["found"] is True
+
+
+def test_different_repositories_on_one_host_stay_separate(tmp_path: Path) -> None:
+    first = _make_repo(tmp_path, "billing")
+    _git(first, "remote", "add", "origin", "git@github.com:acme/billing.git")
+    _save_impl(str(first), _MODEL, "root")
+
+    second = _make_repo(tmp_path, "payments")
+    _git(second, "remote", "add", "origin", "git@github.com:acme/payments.git")
+
+    assert _get_impl(str(second))["found"] is False