mirror of
https://github.com/usestrix/strix.git
synced 2026-08-25 04:12:37 +02:00
fix(safety): allow verdicts without inspection
This commit is contained in:
@@ -53,9 +53,10 @@ issue is resolved and the action is safe. The script runs in a separate networkl
|
||||
container; it cannot inspect the live target or execute commands in the live workspace.
|
||||
|
||||
Do not call the tool when deterministic policy already requires a block or the supplied evidence
|
||||
is already sufficient. When reviewable issues or hard gaps are present, you MUST use run_inspection
|
||||
exactly once before the final verdict. Resolve reviewable issues from the immutable action, source,
|
||||
and input artifacts; do not defer merely because completeness.status is "reviewable".
|
||||
is already sufficient. The inspection call is optional even when reviewable issues or hard gaps are
|
||||
present: use it only when the frozen packet is insufficient for a confident verdict. Resolve
|
||||
reviewable issues from the immutable action, source, and input artifacts; do not defer merely
|
||||
because completeness.status is "reviewable".
|
||||
|
||||
A hard gap is missing evidence, not proof of danger. After inspecting, judge whether the missing
|
||||
evidence could actually change the action's effect, and do not block or defer merely because a gap
|
||||
@@ -254,17 +255,6 @@ class SafetyReviewer:
|
||||
model=model_name,
|
||||
usage=result.context_wrapper.usage,
|
||||
)
|
||||
if (not bundle.complete or bundle.reviewable_issues) and not context.used:
|
||||
return SafetyDecision(
|
||||
allowed=False,
|
||||
source="review_error",
|
||||
reason=(
|
||||
"The reviewer did not use its one inspection call for evidence that required "
|
||||
"correlation."
|
||||
),
|
||||
categories=("missing_evidence_uninspected",),
|
||||
case_id=bundle.case_id,
|
||||
)
|
||||
if context.attempts > 1:
|
||||
return SafetyDecision(
|
||||
allowed=False,
|
||||
|
||||
@@ -537,7 +537,7 @@ async def test_explicit_defer_requires_an_approval_channel(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_interactive_incomplete_evidence_requires_the_inspection_call(
|
||||
async def test_interactive_incomplete_evidence_can_defer_without_inspection(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
@@ -561,21 +561,20 @@ async def test_interactive_incomplete_evidence_requires_the_inspection_call(
|
||||
)
|
||||
|
||||
assert decision.allowed is False
|
||||
assert decision.deferred is False
|
||||
assert decision.categories == ("missing_evidence_uninspected",)
|
||||
assert decision.deferred is True
|
||||
assert decision.source == "reviewer"
|
||||
assert decision.categories == ("incomplete_evidence",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_confident_allow_after_inspection_is_respected_despite_a_hard_gap(
|
||||
async def test_confident_allow_without_inspection_is_respected_despite_a_hard_gap(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
# Once the reviewer has inspected, a confident allow stands even with a hard
|
||||
# gap — it judged the missing evidence irrelevant to the effect (e.g. an
|
||||
# output file that does not exist yet).
|
||||
async def fake_run(_agent: Any, *, context: Any, **_kwargs: Any) -> _Result:
|
||||
context.used = True
|
||||
# A confident allow stands even when the optional inspection is unnecessary:
|
||||
# the packet already proves that the missing file is an output, not an input.
|
||||
async def fake_run(_agent: Any, **_kwargs: Any) -> _Result:
|
||||
return _Result(
|
||||
SafetyVerdict(
|
||||
decision="allow",
|
||||
@@ -589,7 +588,7 @@ async def test_confident_allow_after_inspection_is_respected_despite_a_hard_gap(
|
||||
monkeypatch.setattr(reviewer_module.Runner, "run", fake_run)
|
||||
|
||||
decision = await SafetyReviewer(inspection_runner=_InspectionRunner()).review(
|
||||
_incomplete_bundle(tmp_path, "case-inspected"),
|
||||
_incomplete_bundle(tmp_path, "case-uninspected-allow"),
|
||||
human_approval_available=True,
|
||||
)
|
||||
|
||||
@@ -706,7 +705,7 @@ async def test_collected_workspace_file_can_resolve_hard_gap_and_allow(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_patched_sdk")
|
||||
async def test_reviewable_issue_requires_inspection(
|
||||
async def test_reviewable_issue_can_be_allowed_without_inspection(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
@@ -728,8 +727,9 @@ async def test_reviewable_issue_requires_inspection(
|
||||
_reviewable_bundle(tmp_path, "case-reviewable-uninspected")
|
||||
)
|
||||
|
||||
assert decision.allowed is False
|
||||
assert decision.categories == ("missing_evidence_uninspected",)
|
||||
assert decision.allowed is True
|
||||
assert decision.source == "reviewer"
|
||||
assert decision.reason == "looks safe"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -897,7 +897,7 @@ def test_prompt_judges_security_testing_by_effect_not_technique() -> None:
|
||||
# Ambiguity only reaches a human when an approval channel exists.
|
||||
assert "Return defer only when approval is available" in prompt
|
||||
assert "Without human approval, ambiguity must block" in normalized
|
||||
assert "MUST use run_inspection exactly once" in normalized
|
||||
assert "The inspection call is optional" in normalized
|
||||
# A hard gap is judged by relevance, not blocked outright.
|
||||
assert "A hard gap is missing evidence, not proof of danger" in normalized
|
||||
assert "do not block or defer merely because a gap remains" in normalized
|
||||
|
||||
Reference in New Issue
Block a user