mirror of
https://github.com/usestrix/strix.git
synced 2026-08-25 12:22:37 +02:00
feat(safety): workspace-file reads, approval UX, and integration hardening
Engine + integration: - Reviewer inspection now surfaces the real frozen source of an already-frozen workspace script/dependency instead of an empty string, so workspace-resident scripts resolve without a needless human defer. - Guard effectful static tools via an explicit, documented set plus the SDK's per-tool needs_approval signal; give the exec/stdin wrappers the same idempotency guard as their sibling wrappers. - Centralize DEFAULT_SAFETY_MODE and share one resume safety-mode rule between the CLI and runner so the two cannot drift; type InspectionContext.runner, reuse RUNTIME_STATE_DIR_NAME, and drop a dead workdir parameter and a write-only field. TUI approval experience: - Approve All drops the run into dangerous mode: it approves the pending call and turns review off for the rest of the run, with a standing "review off" status flag. - The status row shows the owning agent as paused while it waits on a decision. - Redesigned prompt: a risk + tool header, a collapsible command/reason preview that expands (e) and scrolls, and no internal digest, agent, or request ids. Full Python (1138) and Go suites, ruff, and mypy strix/ pass. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
41b7b4f392
commit
ccbd8c7b58
@@ -123,6 +123,37 @@ def test_function_tools_are_result_bounded() -> None:
|
||||
assert getattr(by_name["think"], "_strix_bounded", False) is True
|
||||
|
||||
|
||||
def test_only_effectful_static_tools_are_safety_guarded() -> None:
|
||||
# Pins the safety classification of the base tool set: the one effectful
|
||||
# static function tool is guarded for pre-execution review, while internal
|
||||
# bookkeeping and read-only tools run unreviewed. Guarding a read-only tool
|
||||
# would serialize it on the workspace lock and churn other agents' review
|
||||
# epochs, so a new effectful tool must be added to _MUTATING_STATIC_TOOLS.
|
||||
agent = factory.build_strix_agent(is_root=True)
|
||||
by_name = {t.name: t for t in agent.tools}
|
||||
|
||||
assert getattr(by_name["repeat_request"], "_strix_safety_guarded", False) is True
|
||||
for name in ("think", "web_search", "list_requests", "create_note", "view_agent_graph"):
|
||||
assert getattr(by_name[name], "_strix_safety_guarded", False) is False, name
|
||||
|
||||
|
||||
def test_safety_guard_honors_the_sdk_needs_approval_signal() -> None:
|
||||
async def invoke(_ctx: Any, _raw: str) -> str:
|
||||
return "ok"
|
||||
|
||||
future_tool = FunctionTool(
|
||||
name="some_future_effectful_tool",
|
||||
description="test tool",
|
||||
params_json_schema={"type": "object", "properties": {}},
|
||||
on_invoke_tool=invoke,
|
||||
needs_approval=True,
|
||||
)
|
||||
|
||||
guarded = factory._with_safety_guard(future_tool)
|
||||
|
||||
assert getattr(guarded, "_strix_safety_guarded", False) is True
|
||||
|
||||
|
||||
def _capturing_stdin_tool(captured: dict[str, str]) -> FunctionTool:
|
||||
async def invoke(_ctx: Any, raw_input: str) -> str:
|
||||
captured["raw_input"] = raw_input
|
||||
|
||||
@@ -39,6 +39,18 @@ class _Sandbox:
|
||||
return io.BytesIO(self.files[key].encode())
|
||||
|
||||
|
||||
class WorkspaceReadNotFoundError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class _SdkSandbox(_Sandbox):
|
||||
async def read(self, path: Path) -> io.BytesIO:
|
||||
key = path.as_posix()
|
||||
if key not in self.files:
|
||||
raise WorkspaceReadNotFoundError(f"file not found: {key}")
|
||||
return await super().read(path)
|
||||
|
||||
|
||||
def _facts(source: str) -> _PythonFacts:
|
||||
facts = _PythonFacts()
|
||||
facts.visit(ast.parse(source))
|
||||
@@ -115,6 +127,113 @@ async def test_python_script_collects_local_dependency_source() -> None:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_python_path_read_text_collects_literal_input() -> None:
|
||||
hosts_map = "/workspace/recon_infra/hosts_map.txt"
|
||||
bundle = await _compile(
|
||||
"python /workspace/recon.py",
|
||||
{
|
||||
"/workspace/recon.py": (
|
||||
"from pathlib import Path\n"
|
||||
f"HOSTS_MAP = {hosts_map!r}\n"
|
||||
"hosts_path = Path(HOSTS_MAP)\n"
|
||||
"print(hosts_path.read_text())\n"
|
||||
),
|
||||
hosts_map: "admin.example.test\napi.example.test\n",
|
||||
},
|
||||
)
|
||||
try:
|
||||
inputs = [item for item in bundle.packet["artifacts"] if item.get("role") == "input"]
|
||||
assert [item["path"] for item in inputs] == [hosts_map]
|
||||
assert "admin.example.test" in inputs[0]["source"]
|
||||
assert bundle.complete is True
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[
|
||||
"hosts_file = '/workspace/hosts_map.txt'\nopen(hosts_file).read()\n",
|
||||
(
|
||||
"from pathlib import Path\n"
|
||||
"hosts_file = Path('/workspace/hosts_map.txt')\n"
|
||||
"open(hosts_file, 'rb').read()\n"
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_python_open_variable_collects_literal_input(source: str) -> None:
|
||||
bundle = await _compile(
|
||||
"python /workspace/recon.py",
|
||||
{
|
||||
"/workspace/recon.py": source,
|
||||
"/workspace/hosts_map.txt": "one.example.test\n",
|
||||
},
|
||||
)
|
||||
try:
|
||||
inputs = [item for item in bundle.packet["artifacts"] if item.get("role") == "input"]
|
||||
assert [item["path"] for item in inputs] == ["/workspace/hosts_map.txt"]
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
def test_python_write_and_update_modes_are_not_input_dependencies() -> None:
|
||||
facts = _facts(
|
||||
"from pathlib import Path\n"
|
||||
"path = Path('/workspace/output.txt')\n"
|
||||
"open(path, 'w')\n"
|
||||
"open(path, mode='a')\n"
|
||||
"path.open('x')\n"
|
||||
"path.open('r+')\n"
|
||||
)
|
||||
|
||||
assert facts.input_files == set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_not_found_errors_do_not_make_external_imports_incomplete() -> None:
|
||||
ctx = SimpleNamespace(
|
||||
context={
|
||||
"agent_id": "agent-1",
|
||||
"sandbox_session": _SdkSandbox(
|
||||
{"/workspace/check.py": "import json\nfrom pathlib import Path\n"}
|
||||
),
|
||||
},
|
||||
tool_call_id="call-1",
|
||||
turn_input=[],
|
||||
)
|
||||
bundle = await compile_evidence(
|
||||
case_id="case-sdk-not-found",
|
||||
ctx=ctx,
|
||||
arguments={"cmd": "python /workspace/check.py"},
|
||||
mode="guarded",
|
||||
scope={},
|
||||
user_instruction="",
|
||||
settings=SafetySettings(),
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
assert bundle.incomplete_reasons == []
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_relative_import_remains_incomplete() -> None:
|
||||
bundle = await _compile(
|
||||
"python /workspace/pkg/check.py",
|
||||
{"/workspace/pkg/check.py": "from .missing import value\n"},
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is False
|
||||
assert any(
|
||||
"required relative import is missing" in item for item in bundle.incomplete_reasons
|
||||
)
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_automation_inside_script_is_blocked() -> None:
|
||||
bundle = await compile_evidence(
|
||||
@@ -176,7 +295,7 @@ async def test_dynamic_exec_makes_script_evidence_incomplete() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_network_destination_is_incomplete() -> None:
|
||||
async def test_dynamic_network_destination_is_reviewable() -> None:
|
||||
bundle = await compile_evidence(
|
||||
case_id="case-dynamic-network",
|
||||
ctx=_ctx(
|
||||
@@ -188,9 +307,204 @@ async def test_dynamic_network_destination_is_incomplete() -> None:
|
||||
user_instruction="",
|
||||
settings=SafetySettings(),
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
assert bundle.incomplete_reasons == []
|
||||
assert any("dynamic network destination" in item for item in bundle.reviewable_issues)
|
||||
assert bundle.packet["completeness"]["status"] == "reviewable"
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_client_constructor_is_not_a_dynamic_request() -> None:
|
||||
bundle = await _compile(
|
||||
"python /workspace/client.py",
|
||||
{"/workspace/client.py": "import requests\nsession = requests.Session()\n"},
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
assert bundle.packet["artifacts"][0]["dynamic_features"] == []
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
def test_compound_shell_loop_with_script_named_data_is_not_unresolved_execution() -> None:
|
||||
plan = parse_command('for url in app.js; do curl "$url"; done')
|
||||
|
||||
assert plan.compound is True
|
||||
assert plan.parse_error is None
|
||||
|
||||
|
||||
def test_compound_command_with_one_safe_later_script_is_resolved() -> None:
|
||||
plan = parse_command("echo ready && python /workspace/payload.py")
|
||||
|
||||
assert plan.parse_error is None
|
||||
assert plan.script_path == "/workspace/payload.py"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accessible_later_compound_script_is_frozen() -> None:
|
||||
bundle = await _compile(
|
||||
"echo ready && python payload.py",
|
||||
{"/workspace/payload.py": "print('inspected')\n"},
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
assert [item["path"] for item in bundle.packet["artifacts"]] == ["/workspace/payload.py"]
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compound_script_resolves_simple_preceding_cd() -> None:
|
||||
bundle = await _compile(
|
||||
"cd recon && python payload.py",
|
||||
{"/workspace/recon/payload.py": "print('inspected')\n"},
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
assert bundle.packet["artifacts"][0]["path"] == "/workspace/recon/payload.py"
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", ["~", "$HOME", "repo*"])
|
||||
def test_compound_dynamic_or_escaping_cd_is_not_frozen_as_workspace_script(target: str) -> None:
|
||||
plan = parse_command(f"cd {target} && python payload.py")
|
||||
|
||||
assert plan.parse_error is not None
|
||||
assert plan.script_path is None
|
||||
|
||||
|
||||
def test_multiple_compound_script_executions_remain_incomplete() -> None:
|
||||
plan = parse_command("echo ready && python first.py && python second.py")
|
||||
|
||||
assert plan.parse_error is not None
|
||||
assert "issue the script execution separately" in plan.parse_error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command",
|
||||
["cd recon; python payload.py", "printf data | python payload.py"],
|
||||
)
|
||||
def test_ambiguous_compound_script_context_remains_incomplete(command: str) -> None:
|
||||
assert parse_command(command).parse_error is not None
|
||||
|
||||
|
||||
def test_shell_control_prefix_cannot_hide_script_execution() -> None:
|
||||
plan = parse_command("if true; then python /workspace/payload.py; fi")
|
||||
|
||||
assert plan.parse_error is not None
|
||||
assert "issue the script execution separately" in plan.parse_error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command",
|
||||
[
|
||||
"python /workspace/payload.py &",
|
||||
"echo $(python /workspace/payload.py)",
|
||||
"diff <(python /workspace/payload.py) /dev/null",
|
||||
],
|
||||
)
|
||||
def test_background_and_substitution_scripts_are_incomplete(command: str) -> None:
|
||||
assert parse_command(command).parse_error is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_code_shell_substitution_is_reviewable() -> None:
|
||||
bundle = await _compile(
|
||||
"set -e; status=$(curl -s https://example.test); printf '%s' \"$status\""
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
assert bundle.incomplete_reasons == []
|
||||
assert bundle.reviewable_issues == ["shell substitution requires contextual review"]
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_timeout_wrapped_non_script_command_is_reviewable() -> None:
|
||||
bundle = await _compile("timeout 10 curl -s https://example.test")
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
assert bundle.reviewable_issues == ["timeout wrapper requires contextual review"]
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_code_shell_substitution_remains_a_hard_gap() -> None:
|
||||
bundle = await _compile("echo $(python /workspace/payload.py)")
|
||||
try:
|
||||
assert bundle.complete is False
|
||||
assert any("dynamic network destination" in item for item in bundle.incomplete_reasons)
|
||||
assert any("executes code" in item for item in bundle.incomplete_reasons)
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_shell_workspace_script_is_not_reusable_evidence() -> None:
|
||||
bundle = await _compile(
|
||||
"bash -c 'python /workspace/payload.py'",
|
||||
{"/workspace/payload.py": "print('ok')\n"},
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is False
|
||||
assert bundle.workspace_evidence is True
|
||||
assert any("workspace-dependent" in item for item in bundle.incomplete_reasons)
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command",
|
||||
[
|
||||
"if true; then sudo python /workspace/payload.py; fi",
|
||||
"if true; then custom-runner /workspace/payload.py; fi",
|
||||
"case x in x) python /workspace/payload.py;; esac",
|
||||
],
|
||||
)
|
||||
def test_wrapped_script_in_shell_control_flow_is_incomplete(command: str) -> None:
|
||||
plan = parse_command(command)
|
||||
|
||||
assert plan.parse_error is not None
|
||||
assert "issue the script execution separately" in plan.parse_error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[
|
||||
'import requests\nrequests.request("DELETE", target)\n',
|
||||
"import urllib.request\nurllib.request.urlopen(target)\n",
|
||||
"import urllib.request\nurllib.request.urlretrieve(target, '/workspace/out')\n",
|
||||
"import requests.sessions\nrequests.sessions.Session.send(session, prepared)\n",
|
||||
'import httpx\nhttpx.stream("GET", target)\n',
|
||||
"import urllib.request\nurllib.request.OpenerDirector.open(opener, target)\n",
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_request_destination_variants_are_reviewable(source: str) -> None:
|
||||
bundle = await _compile(
|
||||
"python /workspace/client.py",
|
||||
{"/workspace/client.py": source},
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
assert any("dynamic network destination" in item for item in bundle.reviewable_issues)
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_urllib_request_constructor_is_not_a_network_call() -> None:
|
||||
bundle = await _compile(
|
||||
"python /workspace/client.py",
|
||||
{"/workspace/client.py": "import urllib.request\nurllib.request.Request(target)\n"},
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
@@ -345,6 +659,38 @@ async def test_relative_imports_are_collected() -> None:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_relative_imported_attribute_is_not_required_as_a_submodule() -> None:
|
||||
bundle = await _compile(
|
||||
"python /workspace/pkg/main.py",
|
||||
{
|
||||
"/workspace/pkg/main.py": "from .config import VALUE\n",
|
||||
"/workspace/pkg/config.py": "VALUE = 1\n",
|
||||
},
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
paths = {item["path"] for item in bundle.packet["artifacts"]}
|
||||
assert "/workspace/pkg/config.py" in paths
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_package_relative_attribute_is_optional() -> None:
|
||||
bundle = await _compile(
|
||||
"python /workspace/pkg/main.py",
|
||||
{
|
||||
"/workspace/pkg/main.py": "from . import VALUE\n",
|
||||
"/workspace/pkg/__init__.py": "VALUE = 1\n",
|
||||
},
|
||||
)
|
||||
try:
|
||||
assert bundle.complete is True
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_path_mutation_makes_evidence_incomplete() -> None:
|
||||
bundle = await _compile(
|
||||
@@ -848,6 +1194,8 @@ async def test_shell_field_does_not_hide_a_genuine_bash_c_payload() -> None:
|
||||
def test_redirect_input_files_are_parsed_not_heredocs() -> None:
|
||||
assert parse_command("cmd < in.txt").input_files == ["in.txt"]
|
||||
assert parse_command('x < "my hosts.txt" > out.txt').input_files == ["my hosts.txt"]
|
||||
assert parse_command("cmd 3<'fd hosts.txt'").input_files == ["fd hosts.txt"]
|
||||
assert parse_command(r"cmd < escaped\ hosts.txt").input_files == ["escaped hosts.txt"]
|
||||
# A heredoc and a process substitution are not files to read.
|
||||
assert parse_command("cat <<EOF").input_files == []
|
||||
assert parse_command("diff <(a) <(b)").input_files == []
|
||||
@@ -855,6 +1203,15 @@ def test_redirect_input_files_are_parsed_not_heredocs() -> None:
|
||||
assert parse_command("sort f > out.txt").input_files == []
|
||||
|
||||
|
||||
def test_redirect_scanner_ignores_quoted_escaped_and_commented_patterns() -> None:
|
||||
assert parse_command("rg '<form' /workspace/page.html").input_files == []
|
||||
assert parse_command(r"printf \<form").input_files == []
|
||||
assert parse_command("printf ok # < ignored.txt").input_files == []
|
||||
assert parse_command("printf ok # < ignored.txt\ncat < actual.txt").input_files == [
|
||||
"actual.txt"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_workspace_input_file_is_attached_for_action_review() -> None:
|
||||
"""A host list read via `< file` is frozen with the action evidence."""
|
||||
@@ -875,11 +1232,57 @@ async def test_workspace_input_file_is_attached_for_action_review() -> None:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_relative_workdir_is_normalized_for_scripts_inputs_and_packet() -> None:
|
||||
bundle = await _compile(
|
||||
"python recon.py",
|
||||
{
|
||||
"/workspace/repo/recon.py": (
|
||||
"from pathlib import Path\nprint(Path('hosts_map.txt').read_text())\n"
|
||||
),
|
||||
"/workspace/repo/hosts_map.txt": "api.example.test\n",
|
||||
},
|
||||
workdir="repo",
|
||||
)
|
||||
try:
|
||||
assert bundle.packet["pending_action"]["workdir"] == "/workspace/repo"
|
||||
artifacts = bundle.packet["artifacts"]
|
||||
assert artifacts[0]["path"] == "/workspace/repo/recon.py"
|
||||
inputs = [item for item in artifacts if item.get("role") == "input"]
|
||||
assert [item["path"] for item in inputs] == ["/workspace/repo/hosts_map.txt"]
|
||||
assert bundle.complete is True
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tmp_input_file_is_reported_as_unavailable_evidence() -> None:
|
||||
tmp_input = "/tmp/hosts.txt" # noqa: S108 - sandbox fixture path
|
||||
bundle = await _compile(
|
||||
f'while read -r host; do curl "$host"; done < {tmp_input}',
|
||||
{tmp_input: "https://example.test\n"},
|
||||
workdir="/workspace",
|
||||
)
|
||||
try:
|
||||
inputs = [a for a in bundle.packet["artifacts"] if a.get("role") == "input"]
|
||||
assert inputs == []
|
||||
assert bundle.complete is False
|
||||
assert any(
|
||||
"outside the inspectable workspace" in item for item in bundle.incomplete_reasons
|
||||
)
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_file_outside_the_workspace_is_not_read() -> None:
|
||||
bundle = await _compile("cat < /etc/passwd", workdir="/workspace")
|
||||
try:
|
||||
assert [a for a in bundle.packet["artifacts"] if a.get("role") == "input"] == []
|
||||
assert bundle.complete is False
|
||||
assert any(
|
||||
"outside the inspectable workspace" in item for item in bundle.incomplete_reasons
|
||||
)
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
@@ -901,6 +1304,8 @@ async def test_oversize_input_file_is_attached_truncated() -> None:
|
||||
[inp] = [a for a in bundle.packet["artifacts"] if a.get("role") == "input"]
|
||||
assert inp["truncated"] is True
|
||||
assert inp["bytes"] <= settings.max_artifact_bytes
|
||||
assert bundle.complete is False
|
||||
assert any("input file is truncated" in item for item in bundle.incomplete_reasons)
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
@@ -931,12 +1336,50 @@ def test_data_tools_reading_script_named_files_are_not_execution(command: str) -
|
||||
("nuclei --list targets.txt -severity high", ["targets.txt"]),
|
||||
("ffuf -w=words.txt -u https://x/FUZZ", ["words.txt"]),
|
||||
("subfinder -d x -o out.txt", []), # -o is output, not a list input
|
||||
("grep -l pattern /workspace/app.py", []),
|
||||
("curl -w '%{http_code}' https://example.test", []),
|
||||
("nmap -iL /workspace/hosts.txt", ["/workspace/hosts.txt"]),
|
||||
("masscan -iL /workspace/hosts.txt", ["/workspace/hosts.txt"]),
|
||||
("ffuf -w /workspace/words.txt:FUZZ -u https://x/FUZZ", ["/workspace/words.txt"]),
|
||||
],
|
||||
)
|
||||
def test_list_flag_files_are_parsed(command: str, expected: list[str]) -> None:
|
||||
assert parse_command(command).input_files == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("command", "expected"),
|
||||
[
|
||||
("curl --data-binary @/workspace/body.json https://example.test", ["/workspace/body.json"]),
|
||||
("curl --json=@/workspace/body.json https://example.test", ["/workspace/body.json"]),
|
||||
("curl -T /workspace/upload.bin https://example.test", ["/workspace/upload.bin"]),
|
||||
("curl -F file=@/workspace/upload.bin https://example.test", ["/workspace/upload.bin"]),
|
||||
("wget --post-file=/workspace/body.json https://example.test", ["/workspace/body.json"]),
|
||||
(
|
||||
"curl --data-urlencode query@/workspace/body.txt https://example.test",
|
||||
["/workspace/body.txt"],
|
||||
),
|
||||
("http POST https://example.test query@/workspace/body.txt", ["/workspace/body.txt"]),
|
||||
],
|
||||
)
|
||||
def test_request_body_files_are_parsed(command: str, expected: list[str]) -> None:
|
||||
assert parse_command(command).input_files == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_body_file_is_frozen_as_input_evidence() -> None:
|
||||
bundle = await _compile(
|
||||
"curl --data-binary @/workspace/body.json https://example.test",
|
||||
{"/workspace/body.json": '{"probe": true}\n'},
|
||||
)
|
||||
try:
|
||||
inputs = [item for item in bundle.packet["artifacts"] if item.get("role") == "input"]
|
||||
assert [item["path"] for item in inputs] == ["/workspace/body.json"]
|
||||
assert bundle.workspace_evidence is True
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wordlist_flag_file_is_attached_for_action_review() -> None:
|
||||
"""Recon tools route their target list through `-w`/`-l`, not a `<` redirect, so the
|
||||
@@ -955,6 +1398,20 @@ async def test_wordlist_flag_file_is_attached_for_action_review() -> None:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_positional_workspace_data_file_is_attached() -> None:
|
||||
bundle = await _compile(
|
||||
"jq -r '.name' /workspace/recon/cert_names.txt",
|
||||
{"/workspace/recon/cert_names.txt": '{"name":"example.test"}\n'},
|
||||
)
|
||||
try:
|
||||
inputs = [item for item in bundle.packet["artifacts"] if item.get("role") == "input"]
|
||||
assert [item["path"] for item in inputs] == ["/workspace/recon/cert_names.txt"]
|
||||
assert bundle.complete is True
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_flag_value_that_is_not_a_workspace_file_collects_nothing() -> None:
|
||||
"""A boolean `-l` (grep, wc) whose next token is not a workspace file must not make
|
||||
|
||||
@@ -8,7 +8,16 @@ from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
import pytest
|
||||
from agents import Agent, Runner
|
||||
from agents.items import ModelResponse
|
||||
from agents.models.interface import Model
|
||||
from agents.tool_context import ToolContext
|
||||
from agents.usage import Usage
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
)
|
||||
|
||||
import strix.safety.reviewer as reviewer_module
|
||||
from strix.config.settings import SafetySettings
|
||||
@@ -157,6 +166,193 @@ async def test_inspection_tool_can_only_run_once(tmp_path: Path) -> None:
|
||||
assert "inspected" in first
|
||||
assert "already used" in second
|
||||
assert runner.calls == 1
|
||||
assert state.attempts == 2
|
||||
assert state.incomplete is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inspection_collects_workspace_files_before_running_script(tmp_path: Path) -> None:
|
||||
collected: list[tuple[str, ...]] = []
|
||||
|
||||
async def collect(paths: tuple[str, ...]) -> tuple[str, bool]:
|
||||
collected.append(paths)
|
||||
return '{"workspace_artifacts":[{"path":"/workspace/hosts.txt"}]}', False
|
||||
|
||||
runner = _InspectionRunner()
|
||||
state = InspectionContext(
|
||||
evidence_dir=str(tmp_path),
|
||||
runner=runner,
|
||||
collect_workspace=collect,
|
||||
)
|
||||
ctx = ToolContext(
|
||||
context=state,
|
||||
tool_name="run_inspection",
|
||||
tool_call_id="inspect-collect",
|
||||
tool_arguments="{}",
|
||||
)
|
||||
|
||||
result = await run_inspection.on_invoke_tool(
|
||||
ctx,
|
||||
json.dumps(
|
||||
{
|
||||
"reason": "resolve host list",
|
||||
"workspace_paths": ["/workspace/hosts.txt"],
|
||||
"script": "print('analyzed')",
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
assert collected == [("/workspace/hosts.txt",)]
|
||||
assert "workspace_artifacts" in result
|
||||
assert "inspected" in result
|
||||
assert runner.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inspection_can_collect_without_analysis_script(tmp_path: Path) -> None:
|
||||
async def collect(_paths: tuple[str, ...]) -> tuple[str, bool]:
|
||||
return "collected file", False
|
||||
|
||||
state = InspectionContext(
|
||||
evidence_dir=str(tmp_path),
|
||||
runner=_InspectionRunner(),
|
||||
collect_workspace=collect,
|
||||
)
|
||||
ctx = ToolContext(
|
||||
context=state,
|
||||
tool_name="run_inspection",
|
||||
tool_call_id="inspect-read",
|
||||
tool_arguments="{}",
|
||||
)
|
||||
|
||||
result = await run_inspection.on_invoke_tool(
|
||||
ctx,
|
||||
json.dumps(
|
||||
{
|
||||
"reason": "read missing file",
|
||||
"workspace_paths": ["/workspace/missing.txt"],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
assert "collected file" in result
|
||||
assert state.incomplete is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_sdk_loop_replays_inspection_output_into_second_turn(tmp_path: Path) -> None:
|
||||
class LoopModel(Model):
|
||||
def __init__(self) -> None:
|
||||
self.inputs: list[Any] = []
|
||||
self.tool_names: list[list[str]] = []
|
||||
|
||||
async def get_response(self, *_args: Any, **kwargs: Any) -> ModelResponse:
|
||||
self.inputs.append(kwargs["input"])
|
||||
self.tool_names.append([tool.name for tool in kwargs["tools"]])
|
||||
if len(self.inputs) == 1:
|
||||
return ModelResponse(
|
||||
output=[
|
||||
ResponseFunctionToolCall(
|
||||
call_id="inspect-call",
|
||||
name="run_inspection",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"reason": "read host list",
|
||||
"workspace_paths": ["/workspace/hosts.txt"],
|
||||
}
|
||||
),
|
||||
type="function_call",
|
||||
)
|
||||
],
|
||||
usage=Usage(),
|
||||
response_id="response-1",
|
||||
)
|
||||
replay = json.dumps(kwargs["input"], default=str)
|
||||
assert "function_call_output" in replay
|
||||
assert "host-a.example.test" in replay
|
||||
verdict = SafetyVerdict(
|
||||
decision="allow",
|
||||
risk="low",
|
||||
categories=["read_only_reconnaissance"],
|
||||
reason="collected host list proves one bounded GET",
|
||||
confidence=0.99,
|
||||
).model_dump_json()
|
||||
return ModelResponse(
|
||||
output=[
|
||||
ResponseOutputMessage.model_construct(
|
||||
id="message-1",
|
||||
type="message",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
type="output_text",
|
||||
text=verdict,
|
||||
annotations=[],
|
||||
)
|
||||
],
|
||||
)
|
||||
],
|
||||
usage=Usage(),
|
||||
response_id="response-2",
|
||||
)
|
||||
|
||||
def stream_response(self, *_args: Any, **_kwargs: Any) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
async def collect(_paths: tuple[str, ...]) -> tuple[str, bool]:
|
||||
return '{"path":"/workspace/hosts.txt","source":"host-a.example.test"}', False
|
||||
|
||||
model = LoopModel()
|
||||
agent: Agent[InspectionContext] = Agent(
|
||||
name="Safety loop test",
|
||||
instructions="Use the tool once, then return the typed verdict.",
|
||||
model=model,
|
||||
tools=[run_inspection],
|
||||
output_type=SafetyVerdict,
|
||||
tool_use_behavior="run_llm_again",
|
||||
)
|
||||
context = InspectionContext(
|
||||
evidence_dir=str(tmp_path),
|
||||
runner=_InspectionRunner(),
|
||||
collect_workspace=collect,
|
||||
)
|
||||
|
||||
result = await Runner.run(agent, input="deterministic packet", context=context, max_turns=2)
|
||||
|
||||
assert result.final_output_as(SafetyVerdict).decision == "allow"
|
||||
assert model.tool_names == [["run_inspection"], []]
|
||||
assert len(model.inputs) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_repeated_inspection_attempt_fails_review_closed(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
async def fake_run(_agent: Any, *, context: Any, **_kwargs: Any) -> _Result:
|
||||
context.used = True
|
||||
context.attempts = 2
|
||||
return _Result(
|
||||
SafetyVerdict(
|
||||
decision="defer",
|
||||
risk="medium",
|
||||
categories=[],
|
||||
reason="still uncertain",
|
||||
confidence=0.9,
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(reviewer_module.Runner, "run", fake_run)
|
||||
|
||||
decision = await SafetyReviewer(inspection_runner=_InspectionRunner()).review(
|
||||
_incomplete_bundle(tmp_path, "case-repeated-inspection"),
|
||||
human_approval_available=True,
|
||||
)
|
||||
|
||||
assert decision.source == "review_error"
|
||||
assert decision.categories == ("inspection_repeated",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -204,6 +400,38 @@ def _bundle(tmp_path: Path, case_id: str) -> EvidenceBundle:
|
||||
)
|
||||
|
||||
|
||||
def _incomplete_bundle(tmp_path: Path, case_id: str) -> EvidenceBundle:
|
||||
return EvidenceBundle(
|
||||
case_id=case_id,
|
||||
root=tmp_path,
|
||||
packet={
|
||||
"completeness": {
|
||||
"status": "incomplete",
|
||||
"reasons": ["dynamic network destination"],
|
||||
}
|
||||
},
|
||||
complete=False,
|
||||
incomplete_reasons=["dynamic network destination"],
|
||||
)
|
||||
|
||||
|
||||
def _reviewable_bundle(tmp_path: Path, case_id: str) -> EvidenceBundle:
|
||||
return EvidenceBundle(
|
||||
case_id=case_id,
|
||||
root=tmp_path,
|
||||
packet={
|
||||
"completeness": {
|
||||
"status": "reviewable",
|
||||
"hard_gaps": [],
|
||||
"reviewable_issues": ["dynamic network destination"],
|
||||
}
|
||||
},
|
||||
complete=True,
|
||||
incomplete_reasons=[],
|
||||
reviewable_issues=["dynamic network destination"],
|
||||
)
|
||||
|
||||
|
||||
def _verdict_run(verdict: SafetyVerdict) -> Any:
|
||||
async def fake_run(_agent: Any, **_kwargs: Any) -> _Result:
|
||||
return _Result(verdict)
|
||||
@@ -307,6 +535,169 @@ async def test_explicit_defer_requires_an_approval_channel(
|
||||
assert "no human approval channel" in noninteractive.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_interactive_incomplete_evidence_requires_the_inspection_call(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
reviewer_module.Runner,
|
||||
"run",
|
||||
_verdict_run(
|
||||
SafetyVerdict(
|
||||
decision="defer",
|
||||
risk="medium",
|
||||
categories=["incomplete_evidence"],
|
||||
reason="destination remains unknown",
|
||||
confidence=0.9,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
decision = await SafetyReviewer(inspection_runner=_InspectionRunner()).review(
|
||||
_incomplete_bundle(tmp_path, "case-uninspected"),
|
||||
human_approval_available=True,
|
||||
)
|
||||
|
||||
assert decision.allowed is False
|
||||
assert decision.deferred is False
|
||||
assert decision.categories == ("missing_evidence_uninspected",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_incomplete_allow_after_inspection_is_deferred_to_human(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
async def fake_run(_agent: Any, *, context: Any, **_kwargs: Any) -> _Result:
|
||||
context.used = True
|
||||
return _Result(
|
||||
SafetyVerdict(
|
||||
decision="allow",
|
||||
risk="medium",
|
||||
categories=["incomplete_evidence"],
|
||||
reason="available artifacts look non-destructive",
|
||||
confidence=0.95,
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(reviewer_module.Runner, "run", fake_run)
|
||||
|
||||
decision = await SafetyReviewer(inspection_runner=_InspectionRunner()).review(
|
||||
_incomplete_bundle(tmp_path, "case-inspected"),
|
||||
human_approval_available=True,
|
||||
)
|
||||
|
||||
assert decision.allowed is False
|
||||
assert decision.deferred is True
|
||||
assert "dynamic network destination" in decision.reason
|
||||
assert "available artifacts look non-destructive" in decision.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_reviewable_issue_can_be_allowed_after_inspection(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
async def fake_run(_agent: Any, *, context: Any, **_kwargs: Any) -> _Result:
|
||||
context.used = True
|
||||
return _Result(
|
||||
SafetyVerdict(
|
||||
decision="allow",
|
||||
risk="low",
|
||||
categories=["read_only_reconnaissance"],
|
||||
reason="inspection resolved the destination and found fixed GET requests",
|
||||
confidence=0.95,
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(reviewer_module.Runner, "run", fake_run)
|
||||
|
||||
decision = await SafetyReviewer(inspection_runner=_InspectionRunner()).review(
|
||||
_reviewable_bundle(tmp_path, "case-reviewable")
|
||||
)
|
||||
|
||||
assert decision.allowed is True
|
||||
assert decision.deferred is False
|
||||
assert decision.source == "reviewer"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_collected_workspace_file_can_resolve_hard_gap_and_allow(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
bundle = _incomplete_bundle(tmp_path, "case-collected-hard-gap")
|
||||
|
||||
async def collect(paths: tuple[str, ...]) -> tuple[str, bool]:
|
||||
assert paths == ("/workspace/hosts.txt",)
|
||||
bundle.incomplete_reasons.clear()
|
||||
bundle.complete = True
|
||||
bundle.packet["completeness"] = {
|
||||
"status": "complete",
|
||||
"hard_gaps": [],
|
||||
"reviewable_issues": [],
|
||||
}
|
||||
return "collected hosts", False
|
||||
|
||||
async def fake_run(_agent: Any, *, context: Any, **_kwargs: Any) -> _Result:
|
||||
assert context.collect_workspace is not None
|
||||
await context.collect_workspace(("/workspace/hosts.txt",))
|
||||
context.used = True
|
||||
return _Result(
|
||||
SafetyVerdict(
|
||||
decision="allow",
|
||||
risk="low",
|
||||
categories=["read_only_reconnaissance"],
|
||||
reason="collected host list proves bounded GET requests",
|
||||
confidence=0.95,
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(reviewer_module.Runner, "run", fake_run)
|
||||
|
||||
decision = await SafetyReviewer(inspection_runner=_InspectionRunner()).review(
|
||||
bundle,
|
||||
human_approval_available=True,
|
||||
workspace_collector=collect,
|
||||
)
|
||||
|
||||
assert decision.allowed is True
|
||||
assert decision.deferred is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_reviewable_issue_requires_inspection(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
reviewer_module.Runner,
|
||||
"run",
|
||||
_verdict_run(
|
||||
SafetyVerdict(
|
||||
decision="allow",
|
||||
risk="low",
|
||||
categories=[],
|
||||
reason="looks safe",
|
||||
confidence=0.95,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
decision = await SafetyReviewer(inspection_runner=_InspectionRunner()).review(
|
||||
_reviewable_bundle(tmp_path, "case-reviewable-uninspected")
|
||||
)
|
||||
|
||||
assert decision.allowed is False
|
||||
assert decision.categories == ("missing_evidence_uninspected",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_confident_allow_passes(tmp_path: Path, monkeypatch: MonkeyPatch) -> None:
|
||||
@@ -471,9 +862,11 @@ def test_prompt_judges_security_testing_by_effect_not_technique() -> None:
|
||||
assert "solely because their hostname is outside" in prompt
|
||||
# Ambiguity only reaches a human when an approval channel exists.
|
||||
assert "Return defer only when approval is available" in prompt
|
||||
assert "Without human approval, ambiguity must block" in prompt
|
||||
assert "Without human approval, ambiguity must block" in normalized
|
||||
assert "MUST use run_inspection exactly once" in prompt
|
||||
# Non-negotiable guardrails survive.
|
||||
assert 'Never allow when completeness.status is not "complete"' in prompt
|
||||
assert "Never allow when completeness.hard_gaps is non-empty" in prompt
|
||||
assert 'do not defer merely because completeness.status is "reviewable"' in normalized
|
||||
assert "Deterministic policy blocks cannot be overridden" in prompt
|
||||
assert "analysis.mutating_request" in prompt
|
||||
|
||||
|
||||
+408
-38
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import pytest
|
||||
|
||||
from strix.config.settings import SafetySettings
|
||||
from strix.safety.evidence import EvidenceBundle
|
||||
from strix.safety.runtime import SafetyRuntime
|
||||
from strix.safety.types import SafetyApprovalCallback, SafetyApprovalRequest, SafetyDecision
|
||||
|
||||
@@ -58,7 +59,9 @@ class _StubReviewer:
|
||||
bundle: Any,
|
||||
*,
|
||||
human_approval_available: bool = False,
|
||||
workspace_collector: Any = None,
|
||||
) -> SafetyDecision:
|
||||
del workspace_collector
|
||||
self.calls += 1
|
||||
self.human_approval_available.append(human_approval_available)
|
||||
if self.on_review is not None:
|
||||
@@ -224,20 +227,200 @@ async def test_guarded_repeat_request_fails_closed(tmp_path: Path) -> None:
|
||||
|
||||
|
||||
class _Sandbox:
|
||||
def __init__(self) -> None:
|
||||
self.files = {"/workspace/app.py": b"print(1)\n"}
|
||||
|
||||
async def read(self, path: Path) -> io.BytesIO:
|
||||
if path.as_posix() == "/workspace/app.py":
|
||||
return io.BytesIO(b"print(1)\n")
|
||||
raise FileNotFoundError(path)
|
||||
try:
|
||||
return io.BytesIO(self.files[path.as_posix()])
|
||||
except KeyError as exc:
|
||||
raise FileNotFoundError(path) from exc
|
||||
|
||||
|
||||
def _script_ctx() -> Any:
|
||||
class _DirectorySandbox(_Sandbox):
|
||||
async def ls(self, path: Path) -> list[Any]:
|
||||
if path.as_posix() != "/workspace/recon":
|
||||
raise FileNotFoundError(path)
|
||||
return [
|
||||
SimpleNamespace(
|
||||
path="/workspace/recon/hosts.txt",
|
||||
kind=SimpleNamespace(value="file"),
|
||||
size=len(self.files["/workspace/recon/hosts.txt"]),
|
||||
),
|
||||
SimpleNamespace(
|
||||
path="/workspace/recon/probe.py",
|
||||
kind=SimpleNamespace(value="file"),
|
||||
size=len(self.files["/workspace/recon/probe.py"]),
|
||||
),
|
||||
SimpleNamespace(
|
||||
path="/workspace/recon/link",
|
||||
kind=SimpleNamespace(value="symlink"),
|
||||
size=4,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _script_ctx(sandbox: _Sandbox | None = None) -> Any:
|
||||
return SimpleNamespace(
|
||||
context={"agent_id": "agent-1", "sandbox_session": _Sandbox()},
|
||||
context={"agent_id": "agent-1", "sandbox_session": sandbox or _Sandbox()},
|
||||
tool_call_id="call-1",
|
||||
turn_input=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_collector_freezes_requested_workspace_file(tmp_path: Path) -> None:
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
sandbox = _Sandbox()
|
||||
sandbox.files["/workspace/missing.py"] = b"print('safe')\n"
|
||||
evidence_root = tmp_path / "evidence"
|
||||
(evidence_root / "artifacts").mkdir(parents=True)
|
||||
bundle = EvidenceBundle(
|
||||
case_id="case-collect",
|
||||
root=evidence_root,
|
||||
packet={
|
||||
"artifacts": [],
|
||||
"completeness": {
|
||||
"status": "incomplete",
|
||||
"hard_gaps": ["cannot read script entrypoint: /workspace/missing.py"],
|
||||
"reviewable_issues": [],
|
||||
},
|
||||
},
|
||||
complete=False,
|
||||
incomplete_reasons=[
|
||||
"cannot read script entrypoint: /workspace/missing.py",
|
||||
(
|
||||
"compound command executes a script that cannot be frozen as one exact action; "
|
||||
"issue the script execution separately"
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
output, failed = await runtime._collect_workspace_evidence(
|
||||
ctx=_script_ctx(sandbox),
|
||||
bundle=bundle,
|
||||
paths=("/workspace/missing.py",),
|
||||
)
|
||||
|
||||
assert failed is False
|
||||
assert bundle.complete is True
|
||||
assert bundle.workspace_evidence is True
|
||||
assert bundle.incomplete_reasons == []
|
||||
assert bundle.reviewable_issues == [
|
||||
"requested script requires semantic inspection: /workspace/missing.py"
|
||||
]
|
||||
[artifact] = bundle.packet["artifacts"]
|
||||
assert artifact["path"] == "/workspace/missing.py"
|
||||
assert artifact["role"] == "requested_input"
|
||||
assert "sha256:" in output
|
||||
assert "print('safe')" in output
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_already_frozen_script_source_is_surfaced_to_the_reviewer(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
# A script artifact frozen at compile time carries only structural metadata in the
|
||||
# packet; its bytes live on disk under evidence_path. When the reviewer re-requests
|
||||
# that path to resolve a dynamic-destination issue, the collector must hand back the
|
||||
# real source instead of an empty string, or the review defers to a human for a file
|
||||
# it can actually read.
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
evidence_root = tmp_path / "evidence-frozen"
|
||||
(evidence_root / "artifacts").mkdir(parents=True)
|
||||
script_source = 'import requests\nrequests.get("https://example.test/health")\n'
|
||||
(evidence_root / "artifacts" / "000-probe.py").write_text(script_source, encoding="utf-8")
|
||||
bundle = EvidenceBundle(
|
||||
case_id="case-frozen",
|
||||
root=evidence_root,
|
||||
packet={
|
||||
"artifacts": [
|
||||
{
|
||||
"path": "/workspace/probe.py",
|
||||
"digest": "sha256:abc",
|
||||
"bytes": len(script_source),
|
||||
"evidence_path": "artifacts/000-probe.py",
|
||||
}
|
||||
],
|
||||
"completeness": {"status": "reviewable"},
|
||||
},
|
||||
complete=True,
|
||||
incomplete_reasons=[],
|
||||
reviewable_issues=["/workspace/probe.py: dynamic network destination in requests.get"],
|
||||
)
|
||||
|
||||
output, failed = await runtime._collect_workspace_evidence(
|
||||
ctx=_script_ctx(),
|
||||
bundle=bundle,
|
||||
paths=("/workspace/probe.py",),
|
||||
)
|
||||
|
||||
assert failed is False
|
||||
payload = json.loads(output)
|
||||
[result] = payload["workspace_artifacts"]
|
||||
assert result["status"] == "already frozen"
|
||||
assert 'requests.get("https://example.test/health")' in result["source"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_collector_rejects_outside_workspace_path(tmp_path: Path) -> None:
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
evidence_root = tmp_path / "evidence"
|
||||
(evidence_root / "artifacts").mkdir(parents=True)
|
||||
bundle = EvidenceBundle(
|
||||
case_id="case-outside",
|
||||
root=evidence_root,
|
||||
packet={"artifacts": [], "completeness": {}},
|
||||
complete=False,
|
||||
incomplete_reasons=["missing evidence"],
|
||||
)
|
||||
|
||||
output, failed = await runtime._collect_workspace_evidence(
|
||||
ctx=_script_ctx(),
|
||||
bundle=bundle,
|
||||
paths=("/etc/passwd",),
|
||||
)
|
||||
|
||||
assert failed is False
|
||||
assert "outside /workspace" in output
|
||||
assert bundle.packet["artifacts"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_collector_freezes_bounded_workspace_directory(tmp_path: Path) -> None:
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
sandbox = _DirectorySandbox()
|
||||
sandbox.files.update(
|
||||
{
|
||||
"/workspace/recon/hosts.txt": b"a.example.test\n",
|
||||
"/workspace/recon/probe.py": b"print('probe')\n",
|
||||
}
|
||||
)
|
||||
evidence_root = tmp_path / "evidence-tree"
|
||||
(evidence_root / "artifacts").mkdir(parents=True)
|
||||
bundle = EvidenceBundle(
|
||||
case_id="case-tree",
|
||||
root=evidence_root,
|
||||
packet={"artifacts": [], "completeness": {}},
|
||||
complete=True,
|
||||
incomplete_reasons=[],
|
||||
)
|
||||
|
||||
output, failed = await runtime._collect_workspace_evidence(
|
||||
ctx=_script_ctx(sandbox),
|
||||
bundle=bundle,
|
||||
paths=("/workspace/recon/",),
|
||||
)
|
||||
|
||||
assert failed is False
|
||||
assert {item["path"] for item in bundle.packet["artifacts"]} == {
|
||||
"/workspace/recon/hosts.txt",
|
||||
"/workspace/recon/probe.py",
|
||||
}
|
||||
assert "a.example.test" in output
|
||||
assert "symlink" in output
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_stdin_is_blocked_in_guarded_mode(tmp_path: Path) -> None:
|
||||
invoked = False
|
||||
@@ -494,12 +677,8 @@ async def test_defer_without_an_approval_channel_blocks(tmp_path: Path) -> None:
|
||||
assert "no human approval channel" in payload["safety"]["reason"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("command", ["rm -rf /workspace", ""])
|
||||
@pytest.mark.asyncio
|
||||
async def test_deterministic_and_incomplete_blocks_never_request_approval(
|
||||
tmp_path: Path,
|
||||
command: str,
|
||||
) -> None:
|
||||
async def test_deterministic_blocks_never_request_approval(tmp_path: Path) -> None:
|
||||
approval_calls = 0
|
||||
|
||||
async def approve(_request: SafetyApprovalRequest) -> bool:
|
||||
@@ -513,7 +692,7 @@ async def test_deterministic_and_incomplete_blocks_never_request_approval(
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_ctx(),
|
||||
arguments={"cmd": command},
|
||||
arguments={"cmd": "rm -rf /workspace"},
|
||||
invoke_tool=_noop_invoke,
|
||||
)
|
||||
|
||||
@@ -522,6 +701,95 @@ async def test_deterministic_and_incomplete_blocks_never_request_approval(
|
||||
assert approval_calls == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interactive_incomplete_evidence_reaches_reviewer_and_human(tmp_path: Path) -> None:
|
||||
approval_calls = 0
|
||||
|
||||
async def deny(_request: SafetyApprovalRequest) -> bool:
|
||||
nonlocal approval_calls
|
||||
approval_calls += 1
|
||||
return False
|
||||
|
||||
runtime = _runtime(tmp_path, "guarded", deny)
|
||||
reviewer = _StubReviewer(decision=_deferred())
|
||||
runtime._reviewer = reviewer
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_ctx(),
|
||||
arguments={"cmd": ""},
|
||||
invoke_tool=_noop_invoke,
|
||||
)
|
||||
|
||||
payload = json.loads(result)
|
||||
assert payload["status"] == "blocked"
|
||||
assert payload["safety"]["source"] == "human"
|
||||
assert reviewer.calls == 1
|
||||
assert approval_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_headless_incomplete_evidence_still_fails_closed(tmp_path: Path) -> None:
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
reviewer = _StubReviewer(decision=_deferred())
|
||||
runtime._reviewer = reviewer
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_ctx(),
|
||||
arguments={"cmd": ""},
|
||||
invoke_tool=_noop_invoke,
|
||||
)
|
||||
|
||||
payload = json.loads(result)
|
||||
assert payload["safety"]["categories"] == ["incomplete_evidence"]
|
||||
assert reviewer.calls == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_headless_reviewable_uncertainty_reaches_reviewer_and_can_allow(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
sandbox = _Sandbox()
|
||||
sandbox.files["/workspace/app.py"] = b"import requests\nimport sys\nrequests.get(sys.argv[1])\n"
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
reviewer = _StubReviewer()
|
||||
runtime._reviewer = reviewer
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_script_ctx(sandbox),
|
||||
arguments={"cmd": "python /workspace/app.py https://example.test"},
|
||||
invoke_tool=_noop_invoke,
|
||||
)
|
||||
|
||||
assert result == "patched"
|
||||
assert reviewer.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interactive_resolved_reviewable_issue_does_not_prompt_user(tmp_path: Path) -> None:
|
||||
approval_calls = 0
|
||||
|
||||
async def approve(_request: SafetyApprovalRequest) -> bool:
|
||||
nonlocal approval_calls
|
||||
approval_calls += 1
|
||||
return True
|
||||
|
||||
sandbox = _Sandbox()
|
||||
sandbox.files["/workspace/app.py"] = b"import requests\nimport sys\nrequests.get(sys.argv[1])\n"
|
||||
runtime = _runtime(tmp_path, "guarded", approve)
|
||||
reviewer = _StubReviewer()
|
||||
runtime._reviewer = reviewer
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_script_ctx(sandbox),
|
||||
arguments={"cmd": "python /workspace/app.py https://example.test"},
|
||||
invoke_tool=_noop_invoke,
|
||||
)
|
||||
|
||||
assert result == "patched"
|
||||
assert reviewer.calls == 1
|
||||
assert approval_calls == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"decision",
|
||||
[
|
||||
@@ -598,19 +866,16 @@ async def test_review_does_not_hold_the_workspace_lock(tmp_path: Path) -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_workspace_change_during_review_invalidates_the_decision(tmp_path: Path) -> None:
|
||||
async def test_epoch_change_with_unchanged_evidence_executes(tmp_path: Path) -> None:
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
|
||||
async def on_review() -> None:
|
||||
runtime._workspace_epoch += 1
|
||||
|
||||
runtime._reviewer = _StubReviewer(on_review)
|
||||
invoked = False
|
||||
|
||||
async def invoke(_ctx: Any, _raw_input: str) -> str:
|
||||
nonlocal invoked
|
||||
invoked = True
|
||||
return "bad"
|
||||
return "ran"
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_script_ctx(),
|
||||
@@ -618,14 +883,17 @@ async def test_workspace_change_during_review_invalidates_the_decision(tmp_path:
|
||||
invoke_tool=invoke,
|
||||
)
|
||||
|
||||
payload = json.loads(result)
|
||||
assert payload["status"] == "blocked"
|
||||
assert payload["safety"]["categories"] == ["stale_evidence"]
|
||||
assert invoked is False
|
||||
assert result == "ran"
|
||||
assert runtime._reviewer.calls == 1
|
||||
entries = [
|
||||
json.loads(line)
|
||||
for line in (tmp_path / ".state" / "safety-audit.jsonl").read_text().splitlines()
|
||||
]
|
||||
assert any(entry["execution_status"] == "evidence_unchanged" for entry in entries)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_workspace_change_during_human_approval_invalidates_the_decision(
|
||||
async def test_epoch_change_during_human_approval_with_unchanged_evidence_executes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
runtime: SafetyRuntime
|
||||
@@ -636,12 +904,9 @@ async def test_workspace_change_during_human_approval_invalidates_the_decision(
|
||||
|
||||
runtime = _runtime(tmp_path, "guarded", approve)
|
||||
runtime._reviewer = _StubReviewer(decision=_deferred())
|
||||
invoked = False
|
||||
|
||||
async def invoke(_ctx: Any, _raw_input: str) -> str:
|
||||
nonlocal invoked
|
||||
invoked = True
|
||||
return "bad"
|
||||
return "ran"
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_script_ctx(),
|
||||
@@ -649,10 +914,7 @@ async def test_workspace_change_during_human_approval_invalidates_the_decision(
|
||||
invoke_tool=invoke,
|
||||
)
|
||||
|
||||
payload = json.loads(result)
|
||||
assert payload["status"] == "blocked"
|
||||
assert payload["safety"]["categories"] == ["stale_evidence"]
|
||||
assert invoked is False
|
||||
assert result == "ran"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -710,9 +972,7 @@ async def test_guarded_patch_runs_and_advances_the_workspace_epoch(tmp_path: Pat
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_patch_during_review_invalidates_a_script_decision(tmp_path: Path) -> None:
|
||||
"""End-to-end pairing of the two halves: apply_patch bumps the epoch, and a decision
|
||||
compiled before it is refused rather than executed against changed sources."""
|
||||
async def test_noop_patch_during_review_does_not_block_script_execution(tmp_path: Path) -> None:
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
|
||||
async def patch_during_review() -> None:
|
||||
@@ -724,12 +984,9 @@ async def test_a_patch_during_review_invalidates_a_script_decision(tmp_path: Pat
|
||||
)
|
||||
|
||||
runtime._reviewer = _StubReviewer(patch_during_review)
|
||||
invoked = False
|
||||
|
||||
async def invoke(_ctx: Any, _raw_input: str) -> str:
|
||||
nonlocal invoked
|
||||
invoked = True
|
||||
return "bad"
|
||||
return "ran"
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_script_ctx(),
|
||||
@@ -737,9 +994,122 @@ async def test_a_patch_during_review_invalidates_a_script_decision(tmp_path: Pat
|
||||
invoke_tool=invoke,
|
||||
)
|
||||
|
||||
assert result == "ran"
|
||||
assert runtime._reviewer.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_changed_reviewed_file_is_automatically_re_reviewed(tmp_path: Path) -> None:
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
sandbox = _Sandbox()
|
||||
reviews = 0
|
||||
|
||||
async def change_once() -> None:
|
||||
nonlocal reviews
|
||||
reviews += 1
|
||||
if reviews == 1:
|
||||
sandbox.files["/workspace/app.py"] = b"print(2)\n"
|
||||
runtime._workspace_epoch += 1
|
||||
|
||||
runtime._reviewer = _StubReviewer(change_once)
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_script_ctx(sandbox),
|
||||
arguments={"cmd": "python /workspace/app.py"},
|
||||
invoke_tool=_noop_invoke,
|
||||
)
|
||||
|
||||
assert result == "patched"
|
||||
assert runtime._reviewer.calls == 2
|
||||
entries = [
|
||||
json.loads(line)
|
||||
for line in (tmp_path / ".state" / "safety-audit.jsonl").read_text().splitlines()
|
||||
]
|
||||
assert any(entry["execution_status"] == "evidence_changed_re_reviewing" for entry in entries)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_changed_file_after_human_approval_requires_new_approval(tmp_path: Path) -> None:
|
||||
runtime: SafetyRuntime
|
||||
sandbox = _Sandbox()
|
||||
approvals = 0
|
||||
|
||||
async def approve(_request: SafetyApprovalRequest) -> bool:
|
||||
nonlocal approvals
|
||||
approvals += 1
|
||||
if approvals == 1:
|
||||
sandbox.files["/workspace/app.py"] = b"print(2)\n"
|
||||
runtime._workspace_epoch += 1
|
||||
return True
|
||||
|
||||
runtime = _runtime(tmp_path, "guarded", approve)
|
||||
runtime._reviewer = _StubReviewer(decision=_deferred())
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_script_ctx(sandbox),
|
||||
arguments={"cmd": "python /workspace/app.py"},
|
||||
invoke_tool=_noop_invoke,
|
||||
)
|
||||
|
||||
assert result == "patched"
|
||||
assert approvals == 2
|
||||
assert runtime._reviewer.calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unchanged_human_approved_evidence_does_not_prompt_again(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
runtime: SafetyRuntime
|
||||
sandbox = _Sandbox()
|
||||
sandbox.files["/workspace/app.py"] = b"exec(input())\n"
|
||||
approvals = 0
|
||||
|
||||
async def approve(_request: SafetyApprovalRequest) -> bool:
|
||||
nonlocal approvals
|
||||
approvals += 1
|
||||
if approvals == 1:
|
||||
runtime._workspace_epoch += 1
|
||||
return True
|
||||
|
||||
runtime = _runtime(tmp_path, "guarded", approve)
|
||||
runtime._reviewer = _StubReviewer(decision=_deferred())
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_script_ctx(sandbox),
|
||||
arguments={"cmd": "python /workspace/app.py"},
|
||||
invoke_tool=_noop_invoke,
|
||||
)
|
||||
|
||||
assert result == "patched"
|
||||
assert approvals == 1
|
||||
assert runtime._reviewer.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_evidence_churn_stops_after_bounded_re_reviews(tmp_path: Path) -> None:
|
||||
runtime = _runtime(tmp_path, "guarded")
|
||||
sandbox = _Sandbox()
|
||||
changes = 0
|
||||
|
||||
async def change_every_time() -> None:
|
||||
nonlocal changes
|
||||
changes += 1
|
||||
sandbox.files["/workspace/app.py"] = f"print({changes + 1})\n".encode()
|
||||
runtime._workspace_epoch += 1
|
||||
|
||||
runtime._reviewer = _StubReviewer(change_every_time)
|
||||
|
||||
result = await runtime.invoke_exec(
|
||||
ctx=_script_ctx(sandbox),
|
||||
arguments={"cmd": "python /workspace/app.py"},
|
||||
invoke_tool=_noop_invoke,
|
||||
)
|
||||
|
||||
payload = json.loads(result)
|
||||
assert payload["safety"]["categories"] == ["stale_evidence"]
|
||||
assert invoked is False
|
||||
assert payload["status"] == "blocked"
|
||||
assert payload["safety"]["categories"] == ["evidence_churn"]
|
||||
assert runtime._reviewer.calls == 3
|
||||
|
||||
|
||||
async def _noop_invoke(_ctx: Any, _raw_input: str) -> str:
|
||||
|
||||
@@ -392,7 +392,7 @@ async def test_stopping_agent_denies_pending_approvals_for_its_subtree() -> None
|
||||
await controller.handle("agent.stop", {"agent_id": "agent-1"})
|
||||
|
||||
assert await asyncio.gather(*approvals) == ["cancelled", "cancelled"]
|
||||
assert controller.snapshot()["pending_approval"] is None
|
||||
assert controller.snapshot()["pending_approvals"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -417,17 +417,23 @@ async def test_unknown_command_is_rejected() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_safety_approvals_queue_and_resolve_in_order() -> None:
|
||||
async def test_safety_approvals_are_all_visible_and_resolve_independently() -> None:
|
||||
controller = TuiController(args())
|
||||
first = asyncio.create_task(
|
||||
controller.safety_approval_callback(
|
||||
{"request_id": "approval-1", "action": "Run exploit", "reason": "Mutates state"}
|
||||
{
|
||||
"request_id": "approval-1",
|
||||
"agent_id": "agent-1",
|
||||
"action": "Run exploit",
|
||||
"reason": "Mutates state",
|
||||
}
|
||||
)
|
||||
)
|
||||
second = asyncio.create_task(
|
||||
controller.safety_approval_callback(
|
||||
SimpleNamespace(
|
||||
request_id="approval-2",
|
||||
agent_id="agent-2",
|
||||
action="Write a file",
|
||||
reason="Changes the workspace",
|
||||
)
|
||||
@@ -435,35 +441,117 @@ async def test_safety_approvals_queue_and_resolve_in_order() -> None:
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert controller.snapshot()["pending_approval"] == {
|
||||
"request_id": "approval-1",
|
||||
"action": "Run exploit",
|
||||
"reason": "Mutates state",
|
||||
"agent_id": "",
|
||||
"tool_name": "",
|
||||
"digest": "",
|
||||
"risk": "",
|
||||
}
|
||||
assert controller.snapshot()["pending_approvals"] == [
|
||||
{
|
||||
"request_id": "approval-1",
|
||||
"action": "Run exploit",
|
||||
"reason": "Mutates state",
|
||||
"agent_id": "agent-1",
|
||||
"tool_name": "",
|
||||
"digest": "",
|
||||
"risk": "",
|
||||
},
|
||||
{
|
||||
"request_id": "approval-2",
|
||||
"action": "Write a file",
|
||||
"reason": "Changes the workspace",
|
||||
"agent_id": "agent-2",
|
||||
"tool_name": "",
|
||||
"digest": "",
|
||||
"risk": "",
|
||||
},
|
||||
]
|
||||
with pytest.raises(ValueError, match="duplicate safety approval request_id"):
|
||||
await controller.safety_approval_callback(
|
||||
{"request_id": "approval-1", "action": "Duplicate", "reason": "Duplicate"}
|
||||
{
|
||||
"request_id": "approval-1",
|
||||
"agent_id": "agent-1",
|
||||
"action": "Duplicate",
|
||||
"reason": "Duplicate",
|
||||
}
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="stale or unknown"):
|
||||
await controller.handle("safety.resolve", {"request_id": "approval-2", "approved": True})
|
||||
assert await controller.handle(
|
||||
"safety.resolve", {"request_id": "approval-2", "approved": False}
|
||||
) == {"request_id": "approval-2", "approved": False, "approve_all": False}
|
||||
assert await second is False
|
||||
assert [item["request_id"] for item in controller.snapshot()["pending_approvals"]] == [
|
||||
"approval-1"
|
||||
]
|
||||
|
||||
assert await controller.handle(
|
||||
"safety.resolve", {"request_id": "approval-1", "approved": True}
|
||||
) == {"request_id": "approval-1", "approved": True}
|
||||
) == {"request_id": "approval-1", "approved": True, "approve_all": False}
|
||||
assert await first is True
|
||||
assert controller.snapshot()["pending_approval"]["request_id"] == "approval-2"
|
||||
|
||||
with pytest.raises(RuntimeError, match="stale or unknown"):
|
||||
await controller.handle("safety.resolve", {"request_id": "approval-1", "approved": False})
|
||||
await controller.handle("safety.resolve", {"request_id": "approval-2", "approved": False})
|
||||
assert await second is False
|
||||
assert controller.snapshot()["pending_approval"] is None
|
||||
with pytest.raises(RuntimeError, match="No safety approval is pending"):
|
||||
await controller.handle("safety.resolve", {"request_id": "approval-2", "approved": False})
|
||||
assert controller.snapshot()["pending_approvals"] == []
|
||||
|
||||
|
||||
class _RecordingRuntime:
|
||||
def __init__(self) -> None:
|
||||
self.mode = "guarded"
|
||||
|
||||
def disable(self) -> None:
|
||||
self.mode = "off"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approve_all_disables_review_and_releases_the_queue() -> None:
|
||||
controller = TuiController(args())
|
||||
runtime = _RecordingRuntime()
|
||||
controller.register_safety_runtime(runtime)
|
||||
first = asyncio.create_task(
|
||||
controller.safety_approval_callback(
|
||||
{"request_id": "a-1", "agent_id": "agent-1", "action": "Run", "reason": "x"}
|
||||
)
|
||||
)
|
||||
second = asyncio.create_task(
|
||||
controller.safety_approval_callback(
|
||||
{"request_id": "a-2", "agent_id": "agent-2", "action": "Write", "reason": "y"}
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
assert len(controller.snapshot()["pending_approvals"]) == 2
|
||||
|
||||
result = await controller.handle(
|
||||
"safety.resolve", {"request_id": "a-1", "approved": True, "approve_all": True}
|
||||
)
|
||||
|
||||
assert result == {"request_id": "a-1", "approved": True, "approve_all": True}
|
||||
# The chosen call is approved and every other queued call is released as approved.
|
||||
assert await first is True
|
||||
assert await second is True
|
||||
# Review is switched off for the rest of the run and the queue is cleared.
|
||||
assert runtime.mode == "off"
|
||||
assert controller.snapshot()["pending_approvals"] == []
|
||||
# A review already past the runtime's mode check is auto-approved, not queued.
|
||||
later = await controller.safety_approval_callback(
|
||||
{"request_id": "a-3", "agent_id": "agent-1", "action": "Later", "reason": "z"}
|
||||
)
|
||||
assert later is True
|
||||
assert controller.snapshot()["pending_approvals"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approve_all_is_ignored_when_the_answer_is_deny() -> None:
|
||||
controller = TuiController(args())
|
||||
runtime = _RecordingRuntime()
|
||||
controller.register_safety_runtime(runtime)
|
||||
pending = asyncio.create_task(
|
||||
controller.safety_approval_callback(
|
||||
{"request_id": "a-1", "agent_id": "agent-1", "action": "Run", "reason": "x"}
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
result = await controller.handle(
|
||||
"safety.resolve", {"request_id": "a-1", "approved": False, "approve_all": True}
|
||||
)
|
||||
|
||||
assert result == {"request_id": "a-1", "approved": False, "approve_all": False}
|
||||
assert await pending is False
|
||||
# A denial must never flip the run into dangerous mode.
|
||||
assert runtime.mode == "guarded"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -473,6 +561,7 @@ async def test_safety_approval_validates_response_and_sanitizes_display() -> Non
|
||||
controller.safety_approval_callback(
|
||||
{
|
||||
"request_id": "approval-safe",
|
||||
"agent_id": "agent-safe",
|
||||
"action": "run\x1b]52;c;Y2xpcA==\x07 command\x85",
|
||||
"reason": "needs\x1b[31m review\x1b[0m\x7f",
|
||||
}
|
||||
@@ -480,15 +569,17 @@ async def test_safety_approval_validates_response_and_sanitizes_display() -> Non
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert controller.snapshot()["pending_approval"] == {
|
||||
"request_id": "approval-safe",
|
||||
"action": "run command",
|
||||
"reason": "needs review",
|
||||
"agent_id": "",
|
||||
"tool_name": "",
|
||||
"digest": "",
|
||||
"risk": "",
|
||||
}
|
||||
assert controller.snapshot()["pending_approvals"] == [
|
||||
{
|
||||
"request_id": "approval-safe",
|
||||
"action": "run command",
|
||||
"reason": "needs review",
|
||||
"agent_id": "agent-safe",
|
||||
"tool_name": "",
|
||||
"digest": "",
|
||||
"risk": "",
|
||||
}
|
||||
]
|
||||
with pytest.raises(TypeError, match="approved must be a boolean"):
|
||||
await controller.handle(
|
||||
"safety.resolve", {"request_id": "approval-safe", "approved": "yes"}
|
||||
@@ -505,7 +596,12 @@ async def test_safety_approval_validates_response_and_sanitizes_display() -> Non
|
||||
)
|
||||
is False
|
||||
)
|
||||
assert controller.snapshot()["pending_approval"] is None
|
||||
assert controller.snapshot()["pending_approvals"] == []
|
||||
|
||||
with pytest.raises(ValueError, match="agent_id must be a non-empty string"):
|
||||
await controller.safety_approval_callback(
|
||||
{"request_id": "approval-ownerless", "action": "Action", "reason": "Reason"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -513,12 +609,22 @@ async def test_cancelled_safety_request_is_removed_and_reveals_next() -> None:
|
||||
controller = TuiController(args())
|
||||
first = asyncio.create_task(
|
||||
controller.safety_approval_callback(
|
||||
{"request_id": "approval-1", "action": "First", "reason": "First reason"}
|
||||
{
|
||||
"request_id": "approval-1",
|
||||
"agent_id": "agent-1",
|
||||
"action": "First",
|
||||
"reason": "First reason",
|
||||
}
|
||||
)
|
||||
)
|
||||
second = asyncio.create_task(
|
||||
controller.safety_approval_callback(
|
||||
{"request_id": "approval-2", "action": "Second", "reason": "Second reason"}
|
||||
{
|
||||
"request_id": "approval-2",
|
||||
"agent_id": "agent-2",
|
||||
"action": "Second",
|
||||
"reason": "Second reason",
|
||||
}
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
@@ -527,7 +633,7 @@ async def test_cancelled_safety_request_is_removed_and_reveals_next() -> None:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await first
|
||||
|
||||
assert controller.snapshot()["pending_approval"]["request_id"] == "approval-2"
|
||||
assert controller.snapshot()["pending_approvals"][0]["request_id"] == "approval-2"
|
||||
await controller.handle("safety.resolve", {"request_id": "approval-2", "approved": False})
|
||||
assert await second is False
|
||||
|
||||
@@ -538,7 +644,12 @@ async def test_quit_denies_all_pending_and_future_safety_approvals() -> None:
|
||||
requests = [
|
||||
asyncio.create_task(
|
||||
controller.safety_approval_callback(
|
||||
{"request_id": f"approval-{index}", "action": "Action", "reason": "Reason"}
|
||||
{
|
||||
"request_id": f"approval-{index}",
|
||||
"agent_id": f"agent-{index}",
|
||||
"action": "Action",
|
||||
"reason": "Reason",
|
||||
}
|
||||
)
|
||||
)
|
||||
for index in range(2)
|
||||
@@ -548,10 +659,15 @@ async def test_quit_denies_all_pending_and_future_safety_approvals() -> None:
|
||||
await controller.handle("app.quit", {})
|
||||
|
||||
assert await asyncio.gather(*requests) == ["cancelled", "cancelled"]
|
||||
assert controller.snapshot()["pending_approval"] is None
|
||||
assert controller.snapshot()["pending_approvals"] == []
|
||||
assert (
|
||||
await controller.safety_approval_callback(
|
||||
{"request_id": "approval-late", "action": "Late", "reason": "Late reason"}
|
||||
{
|
||||
"request_id": "approval-late",
|
||||
"agent_id": "agent-late",
|
||||
"action": "Late",
|
||||
"reason": "Late reason",
|
||||
}
|
||||
)
|
||||
== "cancelled"
|
||||
)
|
||||
|
||||
@@ -115,6 +115,32 @@ async def receive_initial_state(connection: socket.socket) -> None:
|
||||
complete.add(payload["collection"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_state_frame_can_carry_many_concurrent_approvals() -> None:
|
||||
controller = TuiController(args())
|
||||
requests = [
|
||||
asyncio.create_task(
|
||||
controller.safety_approval_callback(
|
||||
{
|
||||
"request_id": f"approval-{index}",
|
||||
"agent_id": f"agent-{index}",
|
||||
"action": "x" * 500,
|
||||
"reason": "y" * 500,
|
||||
}
|
||||
)
|
||||
)
|
||||
for index in range(80)
|
||||
]
|
||||
await asyncio.sleep(0)
|
||||
server = TuiBackendServer(controller)
|
||||
|
||||
encoded = server._encode(envelope("state", {"revision": 1, "state": controller.snapshot()}))
|
||||
|
||||
assert len(encoded) > MAX_COMMAND_BYTES
|
||||
await controller.cancel_pending_safety_approvals()
|
||||
assert set(await asyncio.gather(*requests)) == {"cancelled"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_requires_ready_before_state_or_commands() -> None:
|
||||
backend, child = socket.socketpair()
|
||||
|
||||
Reference in New Issue
Block a user