mirror of
https://github.com/usestrix/strix.git
synced 2026-08-16 17:27:26 +02:00
Alex Schapiro
d91ac851cf
Promote the coverage ledger from runtime state to a deliverable: coverage.json beside vulnerabilities.json, a Coverage section rendered into the report from the ledger rather than transcribed by an agent, and SARIF pass / notApplicable / open results so a consumer can tell 'tested and clean' from 'never tested'. Ground it in what the runtime observed rather than only what agents claimed: a risk class an agent carried a skill for and never accounted for is published as a gap (and surfaced back to the root agent from finish_scan while it can still act), and a run cut short is stamped incomplete on both the artifact and the SARIF invocation. Also: make the ledger's duplicate check and insertion one critical section and persist under the lock; key a checkout and the URL it was cloned from onto one threat-model identity; render the calibration metadata (counterevidence, confidence, severity change conditions, fix verification) that was being stored and then dropped.
285 lines
9.2 KiB
Python
285 lines
9.2 KiB
Python
"""Tests for the scan coverage ledger."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import pytest
|
|
|
|
from strix.tools.coverage.tools import (
|
|
_list_impl,
|
|
_record_impl,
|
|
_update_impl,
|
|
get_coverage_entries,
|
|
hydrate_coverage_from_disk,
|
|
outcome_counts,
|
|
)
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def coverage_store(tmp_path: Path) -> Path:
|
|
hydrate_coverage_from_disk(tmp_path)
|
|
return tmp_path
|
|
|
|
|
|
def _record(**overrides: str) -> dict[str, Any]:
|
|
kwargs = {
|
|
"surface": "POST /api/orders/{id}",
|
|
"risk_area": "object-level authorization",
|
|
"outcome": "no_issue_found",
|
|
"evidence": "Tested with two tenants; both received 403.",
|
|
"agent_id": "agent-1",
|
|
"agent_name": "authz-tester",
|
|
}
|
|
kwargs.update(overrides)
|
|
return _record_impl(**kwargs)
|
|
|
|
|
|
def test_record_persists_entry(coverage_store: Path) -> None:
|
|
result = _record()
|
|
assert result["success"] is True
|
|
|
|
entries = get_coverage_entries()
|
|
assert len(entries) == 1
|
|
assert entries[0]["surface"] == "POST /api/orders/{id}"
|
|
assert entries[0]["outcome"] == "no_issue_found"
|
|
assert entries[0]["agent_name"] == "authz-tester"
|
|
assert (coverage_store / "coverage.json").exists()
|
|
|
|
|
|
def test_record_normalizes_outcome() -> None:
|
|
assert _record(outcome="Needs Follow-Up")["success"] is True
|
|
assert get_coverage_entries()[0]["outcome"] == "needs_follow_up"
|
|
|
|
|
|
def test_record_rejects_unknown_outcome() -> None:
|
|
result = _record(outcome="looks fine")
|
|
assert result["success"] is False
|
|
assert any("Invalid outcome" in e for e in result["errors"])
|
|
assert not get_coverage_entries()
|
|
|
|
|
|
def test_record_requires_surface_and_risk_area() -> None:
|
|
result = _record(surface=" ", risk_area="")
|
|
assert result["success"] is False
|
|
joined = " ".join(result["errors"])
|
|
assert "surface" in joined
|
|
assert "risk_area" in joined
|
|
|
|
|
|
@pytest.mark.parametrize("outcome", ["ruled_out", "not_applicable", "needs_follow_up"])
|
|
def test_evidence_required_for_asserted_outcomes(outcome: str) -> None:
|
|
result = _record(outcome=outcome, evidence=" ")
|
|
assert result["success"] is False
|
|
assert any("evidence is required" in e for e in result["errors"])
|
|
|
|
|
|
def test_evidence_optional_for_reported() -> None:
|
|
assert _record(outcome="reported", evidence="")["success"] is True
|
|
|
|
|
|
def test_outcome_counts_and_filtering() -> None:
|
|
_record(surface="/login", outcome="reported", evidence="")
|
|
_record(surface="/search", outcome="no_issue_found")
|
|
_record(surface="/upload", outcome="needs_follow_up", evidence="No credentials to test.")
|
|
|
|
assert outcome_counts() == {"reported": 1, "no_issue_found": 1, "needs_follow_up": 1}
|
|
|
|
listed = _list_impl(outcome="needs_follow_up", surface=None, caller_agent_id="agent-1")
|
|
assert listed["filtered_count"] == 1
|
|
assert listed["entries"][0]["surface"] == "/upload"
|
|
assert listed["entries"][0]["by_you"] is True
|
|
|
|
by_surface = _list_impl(outcome=None, surface="sea", caller_agent_id=None)
|
|
assert by_surface["filtered_count"] == 1
|
|
assert by_surface["entries"][0]["surface"] == "/search"
|
|
|
|
|
|
def test_list_rejects_unknown_outcome_filter() -> None:
|
|
result = _list_impl(outcome="bogus", surface=None, caller_agent_id=None)
|
|
assert result["success"] is False
|
|
|
|
|
|
def test_hydrate_reloads_from_disk(coverage_store: Path) -> None:
|
|
_record()
|
|
hydrate_coverage_from_disk(coverage_store)
|
|
entries = get_coverage_entries()
|
|
assert len(entries) == 1
|
|
assert entries[0]["risk_area"] == "object-level authorization"
|
|
|
|
|
|
def _update(entry_id: str, **overrides: str) -> dict[str, Any]:
|
|
kwargs = {
|
|
"entry_id": entry_id,
|
|
"outcome": "reported",
|
|
"evidence": "Got staging credentials and confirmed the IDOR.",
|
|
"agent_id": "agent-2",
|
|
"agent_name": "followup-tester",
|
|
}
|
|
kwargs.update(overrides)
|
|
return _update_impl(**kwargs)
|
|
|
|
|
|
def test_update_moves_outcome_and_keeps_history() -> None:
|
|
recorded = _record(outcome="needs_follow_up", evidence="No credentials to test.")
|
|
entry_id = str(recorded["entry_id"])
|
|
|
|
result = _update(entry_id)
|
|
|
|
assert result["success"] is True
|
|
assert result["previous_outcome"] == "needs_follow_up"
|
|
assert result["outcome"] == "reported"
|
|
|
|
entries = get_coverage_entries()
|
|
assert len(entries) == 1, "update must not create a parallel entry"
|
|
entry = entries[0]
|
|
assert entry["outcome"] == "reported"
|
|
assert entry["agent_name"] == "followup-tester"
|
|
assert entry["history"] == [
|
|
{
|
|
"outcome": "needs_follow_up",
|
|
"recorded_at": entry["created_at"],
|
|
"evidence": "No credentials to test.",
|
|
"agent_name": "authz-tester",
|
|
}
|
|
]
|
|
assert outcome_counts() == {"reported": 1}
|
|
|
|
|
|
def test_update_can_reopen_a_closed_entry() -> None:
|
|
recorded = _record(outcome="ruled_out", evidence="Guard at auth.py:40 covers the path.")
|
|
entry_id = str(recorded["entry_id"])
|
|
|
|
_update(
|
|
entry_id,
|
|
outcome="needs_follow_up",
|
|
evidence="The guard is skipped on the /v2 alias; reachability unproven.",
|
|
)
|
|
|
|
assert outcome_counts() == {"needs_follow_up": 1}
|
|
listed = _list_impl(outcome=None, surface=None, caller_agent_id=None)
|
|
assert listed["entries"][0]["previous_outcomes"] == ["ruled_out"]
|
|
|
|
|
|
def test_update_enforces_evidence_for_closing_outcomes() -> None:
|
|
entry_id = str(_record(outcome="needs_follow_up", evidence="unknown")["entry_id"])
|
|
|
|
result = _update(entry_id, outcome="ruled_out", evidence=" ")
|
|
|
|
assert result["success"] is False
|
|
assert get_coverage_entries()[0]["outcome"] == "needs_follow_up"
|
|
|
|
|
|
def test_update_rejects_unknown_entry() -> None:
|
|
result = _update("nope")
|
|
assert result["success"] is False
|
|
assert "list_coverage" in str(result["error"])
|
|
|
|
|
|
def test_update_persists_to_disk(coverage_store: Path) -> None:
|
|
entry_id = str(_record(outcome="needs_follow_up", evidence="No creds.")["entry_id"])
|
|
_update(entry_id)
|
|
|
|
hydrate_coverage_from_disk(coverage_store)
|
|
|
|
entry = get_coverage_entries()[0]
|
|
assert entry["outcome"] == "reported"
|
|
assert len(entry["history"]) == 1
|
|
|
|
|
|
def test_recording_a_duplicate_surface_is_refused_with_the_existing_id() -> None:
|
|
first = _record_impl(
|
|
surface="/api/invoices",
|
|
risk_area="IDOR",
|
|
outcome="needs_follow_up",
|
|
evidence="No second tenant account to test cross-tenant reads with.",
|
|
agent_id="a1",
|
|
agent_name="Recon",
|
|
)
|
|
|
|
duplicate = _record_impl(
|
|
surface=" /API/Invoices ",
|
|
risk_area="idor",
|
|
outcome="reported",
|
|
evidence="Cross-tenant read confirmed.",
|
|
agent_id="a2",
|
|
agent_name="Authz",
|
|
)
|
|
|
|
assert duplicate["success"] is False
|
|
assert duplicate["existing_entry_id"] == first["entry_id"]
|
|
assert duplicate["existing_outcome"] == "needs_follow_up"
|
|
assert "update_coverage" in duplicate["error"]
|
|
assert len(get_coverage_entries()) == 1
|
|
|
|
|
|
def test_a_different_risk_area_on_one_surface_is_still_its_own_entry() -> None:
|
|
_record_impl(
|
|
surface="/api/invoices",
|
|
risk_area="IDOR",
|
|
outcome="no_issue_found",
|
|
evidence="Tenant id read from the session.",
|
|
agent_id="a1",
|
|
agent_name="Authz",
|
|
)
|
|
second = _record_impl(
|
|
surface="/api/invoices",
|
|
risk_area="SQL injection",
|
|
outcome="no_issue_found",
|
|
evidence="Parameterized throughout.",
|
|
agent_id="a1",
|
|
agent_name="Injection",
|
|
)
|
|
|
|
assert second["success"] is True
|
|
assert len(get_coverage_entries()) == 2
|
|
|
|
|
|
def test_concurrent_records_of_one_surface_yield_a_single_row() -> None:
|
|
"""Duplicate detection and insertion must be one critical section.
|
|
|
|
Two agents recording the same surface at the same moment would otherwise
|
|
both pass the "no duplicate" check, and the report would show a stale
|
|
conclusion beside its replacement — the exact outcome the rejection exists
|
|
to prevent.
|
|
"""
|
|
barrier = threading.Barrier(8)
|
|
|
|
def attempt(index: int) -> dict[str, Any]:
|
|
barrier.wait()
|
|
return _record(agent_id=f"agent-{index}", agent_name=f"tester-{index}")
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
results = list(pool.map(attempt, range(8)))
|
|
|
|
assert sum(1 for result in results if result["success"]) == 1
|
|
assert len(get_coverage_entries()) == 1
|
|
|
|
|
|
def test_concurrent_records_all_survive_persistence(coverage_store: Path) -> None:
|
|
"""A writer holding an older snapshot must not win the rename.
|
|
|
|
If it did, the mirror would come back short on resume and coverage
|
|
recorded before a crash would silently disappear from the report.
|
|
"""
|
|
barrier = threading.Barrier(8)
|
|
|
|
def attempt(index: int) -> dict[str, Any]:
|
|
barrier.wait()
|
|
return _record(surface=f"GET /api/resource/{index}", agent_id=f"agent-{index}")
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
list(pool.map(attempt, range(8)))
|
|
|
|
persisted = json.loads((coverage_store / "coverage.json").read_text(encoding="utf-8"))
|
|
assert len(persisted) == 8
|
|
hydrate_coverage_from_disk(coverage_store)
|
|
assert len(get_coverage_entries()) == 8
|