mirror of
https://github.com/usestrix/strix.git
synced 2026-08-25 04:12:37 +02:00
fix(safety): judge in-scope testing by effect, and stop the shell:bash misread
Two guarded-mode false-positives surfaced in real scan traces. The reviewer blocked a boolean SQL injection probe (`curl "…/login?username='+OR+'1'='1"`) for being an injection attempt at all, though it is a read-only GET that changes nothing. The prompt said "allow only non-destructive" but never established that in-scope offensive testing is the tool's authorized purpose, so the model blocked on the technique. Rewrite the guarded-mode guidance to judge by effect: in-scope injection probes, recon, enumeration, and fuzzing pass, while destructive or persistent effects block — with SQL spelled out (boolean/UNION/time-based read probes pass; DROP, DELETE, INSERT, INTO OUTFILE, stacked statements, and command execution block). Ambiguous evidence still fails closed, and every deterministic block, the completeness gate, observe's passive-only rule, and scope enforcement are kept. Separately the reviewer blocked a plain `curl` as "use of bash shell within a curl command". The shell wrapper stamps `shell: bash` onto every exec_command for execution, and the evidence packet passed that transport default straight to the reviewer, which read it as the agent invoking a shell. Strip the harness-injected transport keys (`shell`, `max_output_tokens`) from the packet's original_arguments; the command itself is still parsed from `cmd`, so an agent-authored `bash -c` payload is unaffected. Note: the effect-based prompt also lets in-scope recon tools (nmap, subfinder, ffuf, katana) through, which the old prompt blocked as "scanning" or "high volume". That follows directly from judging by effect rather than technique. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -797,3 +797,44 @@ def test_imported_attributes_do_not_pollute_the_reported_imports() -> None:
|
||||
|
||||
assert facts.imports == {"os", "mypkg"}
|
||||
assert facts.submodule_imports == {"os.path", "mypkg.CONSTANT"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_harness_transport_keys_are_hidden_from_the_reviewer() -> None:
|
||||
"""The shell wrapper stamps `shell: bash` onto every command; surfacing it in the
|
||||
packet made the reviewer read the transport default as the agent invoking a shell."""
|
||||
bundle = await compile_evidence(
|
||||
case_id="case-transport",
|
||||
ctx=_ctx({}),
|
||||
arguments={
|
||||
"cmd": "curl -I \"https://example.test/login?u='+OR+'1'='1\"",
|
||||
"shell": "bash",
|
||||
"max_output_tokens": 8000,
|
||||
},
|
||||
mode="guarded",
|
||||
scope={"authorized_targets": [{"value": "https://example.test"}]},
|
||||
user_instruction="",
|
||||
settings=SafetySettings(),
|
||||
)
|
||||
try:
|
||||
original = bundle.packet["pending_action"]["original_arguments"]
|
||||
assert "shell" not in original
|
||||
assert "max_output_tokens" not in original
|
||||
assert original["cmd"].startswith("curl")
|
||||
# A GET probe with a boolean payload is not deterministically blocked; the reviewer
|
||||
# judges it by effect.
|
||||
assert bundle.deterministic_block is None
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shell_field_does_not_hide_a_genuine_bash_c_payload() -> None:
|
||||
"""Stripping the transport `shell` key must not weaken parsing of an agent-authored
|
||||
`bash -c`, which is carried in `cmd`, not the shell field."""
|
||||
bundle = await _compile('bash -c "rm -rf /workspace/app"')
|
||||
try:
|
||||
assert bundle.deterministic_block is not None
|
||||
assert "destructive" in bundle.deterministic_block
|
||||
finally:
|
||||
bundle.cleanup()
|
||||
|
||||
@@ -377,3 +377,26 @@ async def test_inspection_failure_output_is_recognized(tmp_path: Path, output: s
|
||||
)
|
||||
|
||||
assert state.incomplete is True
|
||||
|
||||
|
||||
def test_prompt_judges_security_testing_by_effect_not_technique() -> None:
|
||||
"""Pins the effect-based guardrails so a future edit cannot silently revert to
|
||||
blocking in-scope offensive testing on the technique alone."""
|
||||
prompt = reviewer_module._SAFETY_PROMPT
|
||||
|
||||
# Authorization framing and the effect-not-technique rule.
|
||||
assert "authorized penetration test" in prompt
|
||||
assert (
|
||||
'That an action is a "SQL\ninjection"' in prompt
|
||||
or "not, by itself, a reason to block" in prompt
|
||||
)
|
||||
# Read probes pass; writes and destruction block.
|
||||
assert "OR 1=1" in prompt
|
||||
for keyword in ("DROP", "DELETE", "INSERT", "TRUNCATE", "OUTFILE", "xp_cmdshell"):
|
||||
assert keyword in prompt
|
||||
# Fail-closed on ambiguity is preserved.
|
||||
assert "does not settle whether the effect is destructive" in prompt
|
||||
# Non-negotiable guardrails survive.
|
||||
assert 'Never allow when completeness.status is not "complete"' in prompt
|
||||
assert "Deterministic policy blocks cannot be overridden" in prompt
|
||||
assert "analysis.mutating_request is\nnever passive" in prompt
|
||||
|
||||
Reference in New Issue
Block a user