mirror of
https://github.com/usestrix/strix.git
synced 2026-08-22 02:58:39 +02:00
x
This commit is contained in:
@@ -0,0 +1,239 @@
|
||||
"""Tests for the scan coverage ledger."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from strix.tools.coverage.tools import (
|
||||
_list_impl,
|
||||
_record_impl,
|
||||
_update_impl,
|
||||
get_coverage_entries,
|
||||
hydrate_coverage_from_disk,
|
||||
outcome_counts,
|
||||
)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def coverage_store(tmp_path: Path) -> Path:
|
||||
hydrate_coverage_from_disk(tmp_path)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _record(**overrides: str) -> dict[str, object]:
|
||||
kwargs = {
|
||||
"surface": "POST /api/orders/{id}",
|
||||
"risk_area": "object-level authorization",
|
||||
"outcome": "no_issue_found",
|
||||
"evidence": "Tested with two tenants; both received 403.",
|
||||
"agent_id": "agent-1",
|
||||
"agent_name": "authz-tester",
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
return _record_impl(**kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_record_persists_entry(coverage_store: Path) -> None:
|
||||
result = _record()
|
||||
assert result["success"] is True
|
||||
|
||||
entries = get_coverage_entries()
|
||||
assert len(entries) == 1
|
||||
assert entries[0]["surface"] == "POST /api/orders/{id}"
|
||||
assert entries[0]["outcome"] == "no_issue_found"
|
||||
assert entries[0]["agent_name"] == "authz-tester"
|
||||
assert (coverage_store / "coverage.json").exists()
|
||||
|
||||
|
||||
def test_record_normalizes_outcome() -> None:
|
||||
assert _record(outcome="Needs Follow-Up")["success"] is True
|
||||
assert get_coverage_entries()[0]["outcome"] == "needs_follow_up"
|
||||
|
||||
|
||||
def test_record_rejects_unknown_outcome() -> None:
|
||||
result = _record(outcome="looks fine")
|
||||
assert result["success"] is False
|
||||
assert any("Invalid outcome" in e for e in result["errors"]) # type: ignore[union-attr]
|
||||
assert not get_coverage_entries()
|
||||
|
||||
|
||||
def test_record_requires_surface_and_risk_area() -> None:
|
||||
result = _record(surface=" ", risk_area="")
|
||||
assert result["success"] is False
|
||||
joined = " ".join(result["errors"]) # type: ignore[arg-type]
|
||||
assert "surface" in joined
|
||||
assert "risk_area" in joined
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outcome", ["ruled_out", "not_applicable", "needs_follow_up"])
|
||||
def test_evidence_required_for_asserted_outcomes(outcome: str) -> None:
|
||||
result = _record(outcome=outcome, evidence=" ")
|
||||
assert result["success"] is False
|
||||
assert any("evidence is required" in e for e in result["errors"]) # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_evidence_optional_for_reported() -> None:
|
||||
assert _record(outcome="reported", evidence="")["success"] is True
|
||||
|
||||
|
||||
def test_outcome_counts_and_filtering() -> None:
|
||||
_record(surface="/login", outcome="reported", evidence="")
|
||||
_record(surface="/search", outcome="no_issue_found")
|
||||
_record(surface="/upload", outcome="needs_follow_up", evidence="No credentials to test.")
|
||||
|
||||
assert outcome_counts() == {"reported": 1, "no_issue_found": 1, "needs_follow_up": 1}
|
||||
|
||||
listed = _list_impl(outcome="needs_follow_up", surface=None, caller_agent_id="agent-1")
|
||||
assert listed["filtered_count"] == 1
|
||||
assert listed["entries"][0]["surface"] == "/upload" # type: ignore[index]
|
||||
assert listed["entries"][0]["by_you"] is True # type: ignore[index]
|
||||
|
||||
by_surface = _list_impl(outcome=None, surface="sea", caller_agent_id=None)
|
||||
assert by_surface["filtered_count"] == 1
|
||||
assert by_surface["entries"][0]["surface"] == "/search" # type: ignore[index]
|
||||
|
||||
|
||||
def test_list_rejects_unknown_outcome_filter() -> None:
|
||||
result = _list_impl(outcome="bogus", surface=None, caller_agent_id=None)
|
||||
assert result["success"] is False
|
||||
|
||||
|
||||
def test_hydrate_reloads_from_disk(coverage_store: Path) -> None:
|
||||
_record()
|
||||
hydrate_coverage_from_disk(coverage_store)
|
||||
entries = get_coverage_entries()
|
||||
assert len(entries) == 1
|
||||
assert entries[0]["risk_area"] == "object-level authorization"
|
||||
|
||||
|
||||
def _update(entry_id: str, **overrides: str) -> dict[str, object]:
|
||||
kwargs = {
|
||||
"entry_id": entry_id,
|
||||
"outcome": "reported",
|
||||
"evidence": "Got staging credentials and confirmed the IDOR.",
|
||||
"agent_id": "agent-2",
|
||||
"agent_name": "followup-tester",
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
return _update_impl(**kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_update_moves_outcome_and_keeps_history() -> None:
|
||||
recorded = _record(outcome="needs_follow_up", evidence="No credentials to test.")
|
||||
entry_id = str(recorded["entry_id"])
|
||||
|
||||
result = _update(entry_id)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["previous_outcome"] == "needs_follow_up"
|
||||
assert result["outcome"] == "reported"
|
||||
|
||||
entries = get_coverage_entries()
|
||||
assert len(entries) == 1, "update must not create a parallel entry"
|
||||
entry = entries[0]
|
||||
assert entry["outcome"] == "reported"
|
||||
assert entry["agent_name"] == "followup-tester"
|
||||
assert entry["history"] == [
|
||||
{
|
||||
"outcome": "needs_follow_up",
|
||||
"recorded_at": entry["created_at"],
|
||||
"evidence": "No credentials to test.",
|
||||
"agent_name": "authz-tester",
|
||||
}
|
||||
]
|
||||
assert outcome_counts() == {"reported": 1}
|
||||
|
||||
|
||||
def test_update_can_reopen_a_closed_entry() -> None:
|
||||
recorded = _record(outcome="ruled_out", evidence="Guard at auth.py:40 covers the path.")
|
||||
entry_id = str(recorded["entry_id"])
|
||||
|
||||
_update(
|
||||
entry_id,
|
||||
outcome="needs_follow_up",
|
||||
evidence="The guard is skipped on the /v2 alias; reachability unproven.",
|
||||
)
|
||||
|
||||
assert outcome_counts() == {"needs_follow_up": 1}
|
||||
listed = _list_impl(outcome=None, surface=None, caller_agent_id=None)
|
||||
assert listed["entries"][0]["previous_outcomes"] == ["ruled_out"] # type: ignore[index]
|
||||
|
||||
|
||||
def test_update_enforces_evidence_for_closing_outcomes() -> None:
|
||||
entry_id = str(_record(outcome="needs_follow_up", evidence="unknown")["entry_id"])
|
||||
|
||||
result = _update(entry_id, outcome="ruled_out", evidence=" ")
|
||||
|
||||
assert result["success"] is False
|
||||
assert get_coverage_entries()[0]["outcome"] == "needs_follow_up"
|
||||
|
||||
|
||||
def test_update_rejects_unknown_entry() -> None:
|
||||
result = _update("nope")
|
||||
assert result["success"] is False
|
||||
assert "list_coverage" in str(result["error"])
|
||||
|
||||
|
||||
def test_update_persists_to_disk(coverage_store: Path) -> None:
|
||||
entry_id = str(_record(outcome="needs_follow_up", evidence="No creds.")["entry_id"])
|
||||
_update(entry_id)
|
||||
|
||||
hydrate_coverage_from_disk(coverage_store)
|
||||
|
||||
entry = get_coverage_entries()[0]
|
||||
assert entry["outcome"] == "reported"
|
||||
assert len(entry["history"]) == 1
|
||||
|
||||
|
||||
def test_recording_a_duplicate_surface_is_refused_with_the_existing_id() -> None:
|
||||
first = _record_impl(
|
||||
surface="/api/invoices",
|
||||
risk_area="IDOR",
|
||||
outcome="needs_follow_up",
|
||||
evidence="No second tenant account to test cross-tenant reads with.",
|
||||
agent_id="a1",
|
||||
agent_name="Recon",
|
||||
)
|
||||
|
||||
duplicate = _record_impl(
|
||||
surface=" /API/Invoices ",
|
||||
risk_area="idor",
|
||||
outcome="reported",
|
||||
evidence="Cross-tenant read confirmed.",
|
||||
agent_id="a2",
|
||||
agent_name="Authz",
|
||||
)
|
||||
|
||||
assert duplicate["success"] is False
|
||||
assert duplicate["existing_entry_id"] == first["entry_id"]
|
||||
assert duplicate["existing_outcome"] == "needs_follow_up"
|
||||
assert "update_coverage" in duplicate["error"]
|
||||
assert len(get_coverage_entries()) == 1
|
||||
|
||||
|
||||
def test_a_different_risk_area_on_one_surface_is_still_its_own_entry() -> None:
|
||||
_record_impl(
|
||||
surface="/api/invoices",
|
||||
risk_area="IDOR",
|
||||
outcome="no_issue_found",
|
||||
evidence="Tenant id read from the session.",
|
||||
agent_id="a1",
|
||||
agent_name="Authz",
|
||||
)
|
||||
second = _record_impl(
|
||||
surface="/api/invoices",
|
||||
risk_area="SQL injection",
|
||||
outcome="no_issue_found",
|
||||
evidence="Parameterized throughout.",
|
||||
agent_id="a1",
|
||||
agent_name="Injection",
|
||||
)
|
||||
|
||||
assert second["success"] is True
|
||||
assert len(get_coverage_entries()) == 2
|
||||
+35
-1
@@ -8,7 +8,12 @@ from typing import Any
|
||||
import litellm
|
||||
import pytest
|
||||
|
||||
from strix.core.inputs import build_root_task, child_initial_input, make_model_settings
|
||||
from strix.core.inputs import (
|
||||
build_root_task,
|
||||
build_scan_targets,
|
||||
child_initial_input,
|
||||
make_model_settings,
|
||||
)
|
||||
|
||||
|
||||
def _child_kwargs(parent_history: list[Any]) -> dict[str, Any]:
|
||||
@@ -318,3 +323,32 @@ def test_make_model_settings_timeout_survives_reasoning_resolve() -> None:
|
||||
|
||||
assert settings.extra_args is not None
|
||||
assert settings.extra_args["timeout"] == 120.0
|
||||
|
||||
|
||||
def test_scan_targets_prefer_the_workspace_checkout_over_the_remote_url() -> None:
|
||||
config = {
|
||||
"targets": [
|
||||
{
|
||||
"type": "repository",
|
||||
"details": {
|
||||
"target_repo": "https://github.com/acme/billing",
|
||||
"workspace_subdir": "billing",
|
||||
},
|
||||
},
|
||||
{"type": "web_application", "details": {"target_url": "https://app.example.com"}},
|
||||
]
|
||||
}
|
||||
|
||||
assert build_scan_targets(config) == ["/workspace/billing", "https://app.example.com"]
|
||||
|
||||
|
||||
def test_scan_targets_drop_empty_and_duplicate_entries() -> None:
|
||||
config = {
|
||||
"targets": [
|
||||
{"type": "web_application", "details": {"target_url": "https://app.example.com"}},
|
||||
{"type": "web_application", "details": {"target_url": "https://app.example.com"}},
|
||||
{"type": "ip_address", "details": {}},
|
||||
]
|
||||
}
|
||||
|
||||
assert build_scan_targets(config) == ["https://app.example.com"]
|
||||
|
||||
@@ -57,6 +57,9 @@ async def test_create_report_persists_new_fields(report_state: ReportState) -> N
|
||||
remediation_steps="Context-encode output.",
|
||||
evidence="Response echoes the payload verbatim.",
|
||||
assumptions="Assumes a victim opens a crafted link.",
|
||||
counterevidence="No output encoding or CSP observed on this response.",
|
||||
confidence="HIGH",
|
||||
severity_change_conditions="A strict CSP would lower the severity.",
|
||||
fix_effort="LOW",
|
||||
cvss_breakdown=_CVSS,
|
||||
endpoint="/search",
|
||||
@@ -73,6 +76,9 @@ async def test_create_report_persists_new_fields(report_state: ReportState) -> N
|
||||
assert report["fix_effort"] == "low"
|
||||
assert report["fix_pr_body"] == "## Fix\nEncode output."
|
||||
assert report["finding_class"] == "dynamic"
|
||||
assert report["counterevidence"] == "No output encoding or CSP observed on this response."
|
||||
assert report["confidence"] == "high"
|
||||
assert report["severity_change_conditions"] == "A strict CSP would lower the severity."
|
||||
|
||||
|
||||
async def test_create_report_requires_evidence_and_assumptions(
|
||||
@@ -89,6 +95,9 @@ async def test_create_report_requires_evidence_and_assumptions(
|
||||
remediation_steps="r",
|
||||
evidence=" ",
|
||||
assumptions="",
|
||||
counterevidence="none found",
|
||||
confidence="high",
|
||||
severity_change_conditions="n/a",
|
||||
fix_effort="low",
|
||||
cvss_breakdown=_CVSS,
|
||||
endpoint=None,
|
||||
@@ -116,6 +125,9 @@ async def test_create_report_rejects_invalid_fix_effort(report_state: ReportStat
|
||||
remediation_steps="r",
|
||||
evidence="e",
|
||||
assumptions="a",
|
||||
counterevidence="none found",
|
||||
confidence="high",
|
||||
severity_change_conditions="n/a",
|
||||
fix_effort="enormous",
|
||||
cvss_breakdown=_CVSS,
|
||||
endpoint=None,
|
||||
@@ -129,6 +141,80 @@ async def test_create_report_rejects_invalid_fix_effort(report_state: ReportStat
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
async def _create_with(report_state: ReportState, **overrides: object) -> dict[str, object]:
|
||||
kwargs: dict[str, object] = {
|
||||
"title": "X",
|
||||
"description": "d",
|
||||
"impact": "i",
|
||||
"target": "t",
|
||||
"technical_analysis": "ta",
|
||||
"poc_description": "p",
|
||||
"poc_script_code": "c",
|
||||
"remediation_steps": "r",
|
||||
"evidence": "e",
|
||||
"assumptions": "a",
|
||||
"counterevidence": "No guard found on this path.",
|
||||
"confidence": "high",
|
||||
"severity_change_conditions": "Proof of internet exposure would raise it.",
|
||||
"fix_effort": "low",
|
||||
"cvss_breakdown": _CVSS,
|
||||
"endpoint": None,
|
||||
"method": None,
|
||||
"cve": None,
|
||||
"cwe": None,
|
||||
"code_locations": None,
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
assert report_state is not None
|
||||
return await _do_create(**kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
async def test_create_report_requires_counterevidence(report_state: ReportState) -> None:
|
||||
result = await _create_with(report_state, counterevidence=" ")
|
||||
assert result["success"] is False
|
||||
assert any("Counterevidence" in e for e in result["errors"]) # type: ignore[union-attr]
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
async def test_create_report_requires_severity_change_conditions(
|
||||
report_state: ReportState,
|
||||
) -> None:
|
||||
result = await _create_with(report_state, severity_change_conditions="")
|
||||
assert result["success"] is False
|
||||
assert any("severity_change_conditions" in e for e in result["errors"]) # type: ignore[union-attr]
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
async def test_create_report_rejects_invalid_confidence(report_state: ReportState) -> None:
|
||||
result = await _create_with(report_state, confidence="pretty sure")
|
||||
assert result["success"] is False
|
||||
assert any("confidence" in e for e in result["errors"]) # type: ignore[union-attr]
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
async def test_create_report_requires_rationale_when_confidence_not_high(
|
||||
report_state: ReportState,
|
||||
) -> None:
|
||||
result = await _create_with(report_state, confidence="medium")
|
||||
assert result["success"] is False
|
||||
assert any("confidence_rationale" in e for e in result["errors"]) # type: ignore[union-attr]
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
async def test_create_report_accepts_medium_confidence_with_rationale(
|
||||
report_state: ReportState,
|
||||
) -> None:
|
||||
result = await _create_with(
|
||||
report_state,
|
||||
confidence="medium",
|
||||
confidence_rationale="Static-only trace; could not stand up the service.",
|
||||
)
|
||||
assert result["success"] is True
|
||||
report = report_state.vulnerability_reports[0]
|
||||
assert report["confidence"] == "medium"
|
||||
assert report["confidence_rationale"] == "Static-only trace; could not stand up the service."
|
||||
|
||||
|
||||
async def test_dependency_report_sets_class_and_metadata(report_state: ReportState) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2021-23337 in lodash 4.17.20",
|
||||
@@ -562,3 +648,53 @@ def test_vuln_tool_exposes_new_params() -> None:
|
||||
dep_required = create_dependency_report.params_json_schema["required"]
|
||||
assert "package_ecosystem" in dep_required
|
||||
assert "advisory_cvss" in dep_required
|
||||
|
||||
|
||||
_FIX_LOCATION = {
|
||||
"file": "app/views.py",
|
||||
"start_line": 10,
|
||||
"end_line": 12,
|
||||
"fix_before": 'query = f"SELECT * FROM t WHERE id={uid}"',
|
||||
"fix_after": 'query = "SELECT * FROM t WHERE id=%s"',
|
||||
}
|
||||
|
||||
_INFO_LOCATION = {
|
||||
"file": "app/views.py",
|
||||
"start_line": 10,
|
||||
"end_line": 12,
|
||||
"snippet": 'query = f"SELECT * FROM t WHERE id={uid}"',
|
||||
}
|
||||
|
||||
|
||||
async def test_fix_after_requires_verification(report_state: ReportState) -> None:
|
||||
result = await _create_with(report_state, code_locations=[_FIX_LOCATION])
|
||||
assert result["success"] is False
|
||||
assert any("fix_verification" in e for e in result["errors"]) # type: ignore[union-attr]
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
async def test_fix_after_with_verification_persists(report_state: ReportState) -> None:
|
||||
verification = (
|
||||
"Re-ran the PoC against the patched handler: the payload is now bound as a "
|
||||
"parameter and returns no extra rows. Checked the two sibling call sites of "
|
||||
"the same helper and the admin export path; both already parameterized. "
|
||||
"Legitimate numeric ids still resolve and the 404 path is unchanged. "
|
||||
"Ran the focused view tests and ruff."
|
||||
)
|
||||
result = await _create_with(
|
||||
report_state,
|
||||
code_locations=[_FIX_LOCATION],
|
||||
fix_verification=verification,
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert report_state.vulnerability_reports[0]["fix_verification"] == verification
|
||||
|
||||
|
||||
async def test_informational_location_needs_no_verification(report_state: ReportState) -> None:
|
||||
result = await _create_with(report_state, code_locations=[_INFO_LOCATION])
|
||||
assert result["success"] is True
|
||||
assert "fix_verification" not in report_state.vulnerability_reports[0]
|
||||
|
||||
|
||||
def test_vuln_tool_exposes_fix_verification() -> None:
|
||||
assert "fix_verification" in create_vulnerability_report.params_json_schema["properties"]
|
||||
|
||||
@@ -3,6 +3,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
import strix.skills as skills_mod
|
||||
from strix.agents.prompt import _resolve_skills
|
||||
from strix.skills import (
|
||||
get_all_skill_names,
|
||||
get_available_skills,
|
||||
@@ -118,3 +119,42 @@ def test_builtin_skill_still_loads_when_not_overridden(tmp_path: Path) -> None:
|
||||
def test_missing_skill_is_skipped(tmp_path: Path) -> None:
|
||||
register_skill_dir(tmp_path)
|
||||
assert load_skills(["does_not_exist"]) == {}
|
||||
|
||||
|
||||
def test_resolve_skills_always_includes_analysis_baseline() -> None:
|
||||
resolved = _resolve_skills(requested=None)
|
||||
|
||||
assert "analysis/counterevidence" in resolved
|
||||
assert "analysis/severity_calibration" in resolved
|
||||
|
||||
|
||||
def test_resolve_skills_adds_diff_mode_only_when_diff_scoped() -> None:
|
||||
assert "scan_modes/diff" not in _resolve_skills(requested=None)
|
||||
diff_scoped = _resolve_skills(requested=None, is_diff_scoped=True)
|
||||
assert "scan_modes/diff" in diff_scoped
|
||||
# Diff scope overlays the depth mode rather than replacing it.
|
||||
assert "scan_modes/deep" in diff_scoped
|
||||
|
||||
|
||||
def test_resolve_skills_gates_source_aware_skills_on_whitebox() -> None:
|
||||
blackbox = _resolve_skills(requested=None)
|
||||
assert "analysis/fix_verification" not in blackbox
|
||||
assert "analysis/source_aware_discovery" not in blackbox
|
||||
|
||||
whitebox = _resolve_skills(requested=None, is_whitebox=True)
|
||||
assert "analysis/fix_verification" in whitebox
|
||||
assert "analysis/source_aware_discovery" in whitebox
|
||||
|
||||
|
||||
def test_new_skill_files_load() -> None:
|
||||
names = [
|
||||
"analysis/counterevidence",
|
||||
"analysis/severity_calibration",
|
||||
"analysis/fix_verification",
|
||||
"analysis/source_aware_discovery",
|
||||
"scan_modes/diff",
|
||||
]
|
||||
loaded = load_skills(names)
|
||||
for name in names:
|
||||
key = name.split("/")[-1]
|
||||
assert loaded.get(key), f"{name} failed to load"
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Tests for the target-scoped threat model cache."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from strix.agents.factory import _BASE_TOOLS
|
||||
from strix.tools.threat_model import tools as threat_model_tools
|
||||
from strix.tools.threat_model.tools import (
|
||||
_amend_impl,
|
||||
_get_impl,
|
||||
_save_impl,
|
||||
amend_threat_model,
|
||||
get_threat_model,
|
||||
save_threat_model,
|
||||
)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_MODEL = """# Threat Model
|
||||
|
||||
## Overview
|
||||
A multi-tenant billing API. Product code lives in `api/`; `scripts/` is
|
||||
developer-only tooling and is not deployed.
|
||||
|
||||
## Trust Boundaries and Assumptions
|
||||
Requests arrive from untrusted tenants through `api/router.py`. The tenant id
|
||||
is taken from the signed session, never from the request body. Operators
|
||||
configure webhooks; developers control migrations.
|
||||
|
||||
## Attack Surface and Attacker Stories
|
||||
The public REST surface and the webhook receiver are attacker-reachable. A
|
||||
realistic story is a tenant reading another tenant's invoices. Local CLI
|
||||
tooling is not a realistic surface.
|
||||
|
||||
## Severity Calibration
|
||||
Critical: cross-tenant write. High: cross-tenant read. Medium: authenticated
|
||||
self-scoped information leak. Low: verbose errors.
|
||||
"""
|
||||
|
||||
|
||||
def _git(repo: Path, *args: str) -> None:
|
||||
subprocess.run(["/usr/bin/env", "git", *args], cwd=repo, check=True) # noqa: S603
|
||||
|
||||
|
||||
def _make_repo(tmp_path: Path, name: str = "repo") -> Path:
|
||||
repo = tmp_path / name
|
||||
repo.mkdir(parents=True)
|
||||
_git(repo, "init", "-q")
|
||||
_git(repo, "config", "user.email", "t@example.com")
|
||||
_git(repo, "config", "user.name", "t")
|
||||
(repo / "README.md").write_text("hi\n", encoding="utf-8")
|
||||
_git(repo, "add", "README.md")
|
||||
_git(repo, "commit", "-qm", "init")
|
||||
return repo
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(threat_model_tools, "_CACHE_DIR", tmp_path / "cache")
|
||||
|
||||
|
||||
def test_missing_model_reports_not_found(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
|
||||
result = _get_impl(str(repo))
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["found"] is False
|
||||
assert "save_threat_model" in result["message"]
|
||||
|
||||
|
||||
def test_saved_model_round_trips(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
|
||||
assert _save_impl(str(repo), _MODEL, "Strix")["success"] is True
|
||||
result = _get_impl(str(repo))
|
||||
|
||||
assert result["found"] is True
|
||||
assert result["stale"] is False
|
||||
assert "multi-tenant billing API" in result["content"]
|
||||
|
||||
|
||||
def test_model_is_stale_after_new_revision(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
_save_impl(str(repo), _MODEL, None)
|
||||
|
||||
(repo / "next.py").write_text("x = 1\n", encoding="utf-8")
|
||||
_git(repo, "add", "next.py")
|
||||
_git(repo, "commit", "-qm", "next")
|
||||
|
||||
result = _get_impl(str(repo))
|
||||
|
||||
assert result["found"] is True
|
||||
assert result["stale"] is True
|
||||
assert result["content"]
|
||||
|
||||
|
||||
def test_cache_is_keyed_per_repository(tmp_path: Path) -> None:
|
||||
first = _make_repo(tmp_path, "first")
|
||||
second = _make_repo(tmp_path, "second")
|
||||
_save_impl(str(first), _MODEL, None)
|
||||
|
||||
assert _get_impl(str(second))["found"] is False
|
||||
|
||||
|
||||
def test_rejects_model_missing_required_sections(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
thin = _MODEL.replace("## Severity Calibration", "## Notes")
|
||||
|
||||
result = _save_impl(str(repo), thin, None)
|
||||
|
||||
assert result["success"] is False
|
||||
assert "severity calibration" in result["error"]
|
||||
|
||||
|
||||
def test_rejects_stub_model(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
|
||||
result = _save_impl(str(repo), "overview trust boundaries attack surface", None)
|
||||
|
||||
assert result["success"] is False
|
||||
assert "too thin" in result["error"]
|
||||
|
||||
|
||||
def test_rejects_empty_target() -> None:
|
||||
result = _get_impl(" ")
|
||||
assert result["success"] is False
|
||||
assert "target cannot be empty" in result["error"]
|
||||
|
||||
|
||||
def test_tools_are_registered() -> None:
|
||||
assert get_threat_model in _BASE_TOOLS
|
||||
assert save_threat_model in _BASE_TOOLS
|
||||
|
||||
|
||||
_ADDENDUM = (
|
||||
"The base model calls the webhook receiver operator-controlled. It is "
|
||||
"unauthenticated in `api/webhooks.py:31`, so treat its body as attacker-controlled."
|
||||
)
|
||||
|
||||
|
||||
def test_amendment_is_returned_with_the_model(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
_save_impl(str(repo), _MODEL, "root")
|
||||
|
||||
assert _amend_impl(str(repo), _ADDENDUM, "webhook-agent")["success"] is True
|
||||
result = _get_impl(str(repo))
|
||||
|
||||
assert result["content"] == _MODEL.strip()
|
||||
assert [a["content"] for a in result["amendments"]] == [_ADDENDUM]
|
||||
assert result["amendments"][0]["by"] == "webhook-agent"
|
||||
|
||||
|
||||
def test_amendments_accumulate_without_overwriting(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
_save_impl(str(repo), _MODEL, "root")
|
||||
|
||||
_amend_impl(str(repo), _ADDENDUM, "agent-a")
|
||||
second = "The `scripts/` directory ships in the container image; it is not dev-only."
|
||||
_amend_impl(str(repo), second + " See `Dockerfile:14`.", "agent-b")
|
||||
|
||||
amendments = _get_impl(str(repo))["amendments"]
|
||||
assert len(amendments) == 2
|
||||
assert [a["by"] for a in amendments] == ["agent-a", "agent-b"]
|
||||
|
||||
|
||||
def test_amend_requires_an_existing_model(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
|
||||
result = _amend_impl(str(repo), _ADDENDUM, None)
|
||||
|
||||
assert result["success"] is False
|
||||
assert "save_threat_model" in result["error"]
|
||||
|
||||
|
||||
def test_amend_rejects_a_stub(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
_save_impl(str(repo), _MODEL, "root")
|
||||
|
||||
assert _amend_impl(str(repo), "looks wrong", None)["success"] is False
|
||||
|
||||
|
||||
def test_save_clears_amendments_and_says_so(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
_save_impl(str(repo), _MODEL, "root")
|
||||
_amend_impl(str(repo), _ADDENDUM, "agent-a")
|
||||
|
||||
result = _save_impl(str(repo), _MODEL.replace("billing API", "billing service"), "root")
|
||||
|
||||
assert result["amendments_cleared"] == 1
|
||||
assert "cleared" in result["message"]
|
||||
assert "amendments" not in _get_impl(str(repo))
|
||||
|
||||
|
||||
def test_amend_tool_is_registered() -> None:
|
||||
assert amend_threat_model in _BASE_TOOLS
|
||||
|
||||
|
||||
_BLACKBOX_MODEL = _MODEL.replace(
|
||||
"Product code lives in `api/`; `scripts/` is\ndeveloper-only tooling and is not deployed.",
|
||||
"Only the deployed surface is visible; no source. Inferred from recon.",
|
||||
)
|
||||
|
||||
|
||||
def test_blackbox_target_round_trips() -> None:
|
||||
target = "https://app.example.com"
|
||||
|
||||
assert _save_impl(target, _BLACKBOX_MODEL, "recon")["success"] is True
|
||||
result = _get_impl(target)
|
||||
|
||||
assert result["found"] is True
|
||||
assert result["stale"] is False, "a fresh model with no revision is not stale"
|
||||
assert result["revision"] == "unversioned"
|
||||
assert "Inferred from recon" in result["content"]
|
||||
|
||||
|
||||
def test_blackbox_target_spellings_share_one_model() -> None:
|
||||
_save_impl("https://App.Example.com:443/", _BLACKBOX_MODEL, "recon")
|
||||
|
||||
for spelling in ("https://app.example.com", "app.example.com", "https://app.example.com/"):
|
||||
assert _get_impl(spelling)["found"] is True, spelling
|
||||
|
||||
assert _get_impl("https://other.example.com")["found"] is False
|
||||
|
||||
|
||||
def test_blackbox_model_goes_stale_with_age() -> None:
|
||||
target = "https://app.example.com"
|
||||
_save_impl(target, _BLACKBOX_MODEL, "recon")
|
||||
|
||||
aged = (datetime.now(UTC) - timedelta(days=threat_model_tools._MAX_AGE_DAYS + 1)).isoformat()
|
||||
path = threat_model_tools._cache_path("app.example.com:443")
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
payload["created_at"] = aged
|
||||
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
result = _get_impl(target)
|
||||
|
||||
assert result["stale"] is True
|
||||
assert "re-confirm" in result["message"]
|
||||
|
||||
|
||||
def test_blackbox_target_can_be_amended() -> None:
|
||||
target = "https://app.example.com"
|
||||
_save_impl(target, _BLACKBOX_MODEL, "recon")
|
||||
|
||||
addendum = (
|
||||
"The model infers /admin is IP-restricted. It is reachable with any "
|
||||
"authenticated session; the restriction is only on /admin/settings."
|
||||
)
|
||||
assert _amend_impl(target, addendum, "authz-agent")["success"] is True
|
||||
assert _get_impl(target)["amendments"][0]["content"] == addendum
|
||||
|
||||
|
||||
def test_checkout_and_its_remote_are_the_same_target(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
_git(repo, "remote", "add", "origin", "https://github.com/acme/billing.git")
|
||||
_save_impl(str(repo), _MODEL, "root")
|
||||
|
||||
clone = _make_repo(tmp_path, "clone")
|
||||
_git(clone, "remote", "add", "origin", "https://github.com/acme/billing.git")
|
||||
|
||||
assert _get_impl(str(clone))["found"] is True
|
||||
|
||||
|
||||
def test_path_on_a_known_host_resolves_to_the_scan_target() -> None:
|
||||
scan_targets = ["https://app.example.com"]
|
||||
_save_impl("https://app.example.com", _BLACKBOX_MODEL, "root", scan_targets)
|
||||
|
||||
# An agent testing one page names that page, not the scan's target string.
|
||||
assert _get_impl("https://app.example.com/admin/login", scan_targets)["found"] is True
|
||||
|
||||
|
||||
def test_two_scan_targets_on_one_host_stay_separate() -> None:
|
||||
scan_targets = ["https://example.com/tenant-a", "https://example.com/tenant-b"]
|
||||
_save_impl("https://example.com/tenant-a", _BLACKBOX_MODEL, "root", scan_targets)
|
||||
|
||||
assert _get_impl("https://example.com/tenant-b", scan_targets)["found"] is False
|
||||
|
||||
|
||||
def test_unknown_host_is_not_snapped_onto_the_scan_target() -> None:
|
||||
scan_targets = ["https://app.example.com"]
|
||||
_save_impl("https://app.example.com", _BLACKBOX_MODEL, "root", scan_targets)
|
||||
|
||||
assert _get_impl("https://unrelated.test", scan_targets)["found"] is False
|
||||
|
||||
|
||||
def test_empty_target_falls_back_to_a_single_scan_target() -> None:
|
||||
scan_targets = ["https://app.example.com"]
|
||||
_save_impl("", _BLACKBOX_MODEL, "root", scan_targets)
|
||||
|
||||
assert _get_impl("", scan_targets)["found"] is True
|
||||
assert _get_impl("https://app.example.com")["found"] is True
|
||||
|
||||
|
||||
def test_repository_subdirectory_shares_the_repository_model(tmp_path: Path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
(repo / "src").mkdir()
|
||||
_save_impl(str(repo), _MODEL, "root")
|
||||
|
||||
assert _get_impl(str(repo / "src"))["found"] is True
|
||||
Reference in New Issue
Block a user