Compare commits

..
Author SHA1 Message Date
Alex Schapiro d31d6fb6b8 fix(runtime): retry transient sandbox startup failures 2026-07-15 02:59:55 +00:00
alex sandGitHub d44ca88a18 fix(runtime): stage symlink-safe copies for LocalDir uploads (#766)
The sandbox SDK's LocalDir walker rejects any symlink outright
(LocalDirReadError, reason=symlink_not_supported), so uploading a cloned
repository that commits symlinks (common in JS/TS monorepos) aborts before
the agent starts. Stage such trees into a temp copy first: in-tree links
are dereferenced; out-of-tree, dangling, and cyclic links are dropped and
never followed, preserving the walker's path-escape safety. Symlink-free
trees are uploaded as-is.
2026-07-14 17:40:23 -04:00
6 changed files with 599 additions and 18 deletions
+130 -3
View File
@@ -2,7 +2,9 @@
from __future__ import annotations
import asyncio
import logging
import os
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Any
@@ -16,6 +18,127 @@ logger = logging.getLogger(__name__)
SandboxBackend = Callable[..., Awaitable[tuple[Any, Any]]]
_DEFAULT_START_ATTEMPTS = 3
_START_BACKOFF_SECONDS = 2.0
_TRANSIENT_TIMEOUT_NAMES = {
"ConnectTimeout",
"PoolTimeout",
"ReadTimeout",
"TimeoutError",
"TimeoutException",
"WriteTimeout",
}
_TRANSIENT_CONNECTION_NAMES = {
"ConnectError",
"ConnectionError",
"ConnectionResetError",
"ReadError",
"WriteError",
}
def _start_attempts() -> int:
raw = os.environ.get("STRIX_E2B_BOOTSTRAP_ATTEMPTS")
if raw is None:
return _DEFAULT_START_ATTEMPTS
try:
attempts = int(raw)
except ValueError:
logger.warning(
"Invalid STRIX_E2B_BOOTSTRAP_ATTEMPTS=%r; using %d",
raw,
_DEFAULT_START_ATTEMPTS,
)
return _DEFAULT_START_ATTEMPTS
if attempts < 1:
logger.warning(
"STRIX_E2B_BOOTSTRAP_ATTEMPTS must be positive; using %d",
_DEFAULT_START_ATTEMPTS,
)
return _DEFAULT_START_ATTEMPTS
return attempts
def _exception_chain(error: BaseException) -> list[BaseException]:
chain: list[BaseException] = []
pending: list[BaseException | None] = [error]
seen: set[int] = set()
while pending:
current = pending.pop()
if current is None or id(current) in seen:
continue
seen.add(id(current))
chain.append(current)
pending.extend(
(
current.__cause__,
current.__context__,
getattr(current, "cause", None),
)
)
return chain
def _is_transient_start_error(error: BaseException) -> bool:
for cause in _exception_chain(error):
name = type(cause).__name__
module = type(cause).__module__
if isinstance(cause, TimeoutError | ConnectionError | ConnectionResetError):
return True
if name in _TRANSIENT_TIMEOUT_NAMES:
return True
if name in _TRANSIENT_CONNECTION_NAMES and (
module.startswith(("httpcore", "httpx", "e2b", "agents"))
or name in {"ConnectionError", "ConnectionResetError"}
):
return True
return False
async def start_session_with_retry(
client: Any,
create_session: Callable[[], Awaitable[Any]],
*,
attempts: int | None = None,
) -> Any:
"""Start a sandbox session, retrying transient transport failures.
Backend implementations should use this helper when they own both session
creation and ``session.start()`` so failed starts can be torn down before a
retry. The caller owns the manifest and any temporary source directories
until this helper returns.
"""
max_attempts = attempts if attempts is not None else _start_attempts()
for attempt in range(1, max_attempts + 1):
session: Any | None = None
try:
session = await create_session()
assert session is not None
await session.start()
except Exception as exc:
if session is not None:
try:
await client.delete(session)
except Exception: # noqa: BLE001
logger.warning(
"Failed to tear down sandbox after start failure",
exc_info=True,
)
transient = _is_transient_start_error(exc)
if not transient or attempt == max_attempts:
raise
delay = _START_BACKOFF_SECONDS * (2 ** (attempt - 1))
logger.warning(
"Transient sandbox start failure; retrying attempt %d/%d in %.1fs",
attempt + 1,
max_attempts,
delay,
)
await asyncio.sleep(delay)
else:
return session
raise AssertionError("sandbox start retry loop completed without returning or raising")
async def _docker_backend(
*,
@@ -50,8 +173,10 @@ async def _docker_backend(
client = StrixDockerSandboxClient(docker.from_env())
client.strix_bind_mounts = bind_mounts or []
options = DockerSandboxClientOptions(image=image, exposed_ports=exposed_ports)
session = await client.create(options=options, manifest=manifest)
await session.start()
session = await start_session_with_retry(
client,
lambda: client.create(options=options, manifest=manifest),
)
return client, session
@@ -83,7 +208,9 @@ def register_backend(name: str, backend: SandboxBackend) -> None:
Intended for downstream users who ship their own runtime — register
before any ``session_manager.create_or_reuse`` call. Re-registering
an existing name overwrites the prior entry.
an existing name overwrites the prior entry. Backends that own both
session creation and ``session.start()`` should use
:func:`start_session_with_retry`.
"""
_BACKENDS[name] = backend
logger.info("Registered sandbox backend: %s", name)
+120
View File
@@ -0,0 +1,120 @@
"""Symlink-safe staging for ``LocalDir`` manifest uploads.
The sandbox SDK's ``LocalDir`` walker refuses to copy symlinks at all — it
raises ``LocalDirReadError(reason="symlink_not_supported")`` on the first one
as a path-escape / TOCTOU safeguard. Real source trees (especially JS/TS
monorepos with workspace or shared-config links) routinely commit symlinks, so
handing such a tree straight to ``LocalDir`` aborts the upload before the agent
even starts.
:func:`stage_symlink_safe_dir` returns a path that is always safe to hand to
``LocalDir``:
* a tree with no symlinks is used as-is (no copy);
* otherwise the tree is copied into a temp directory with symlinks resolved:
- a link whose target stays inside the tree is *dereferenced* (its target
content is materialized in place), so the agent still sees the file;
- a link that escapes the tree, dangles, or forms a cycle is *dropped* and
never followed. Refusing to follow out-of-tree links preserves the walker's
path-escape safety and keeps host/out-of-tree content from leaking into the
(hostile) sandbox.
Regular files are hard-linked when possible (falling back to a copy across
devices), so the staged tree adds negligible disk for the non-symlink bulk.
"""
from __future__ import annotations
import logging
import os
import shutil
import tempfile
from pathlib import Path
logger = logging.getLogger(__name__)
_STAGING_PREFIX = "strix-localdir-"
def _is_within(target: Path, root: Path) -> bool:
"""Return whether ``target`` is ``root`` itself or nested under it."""
if target == root:
return True
try:
target.relative_to(root)
except ValueError:
return False
return True
def tree_has_symlink(root: Path) -> bool:
"""Return whether ``root`` contains any symlink (file or directory)."""
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
base = Path(dirpath)
for name in (*dirnames, *filenames):
if (base / name).is_symlink():
return True
return False
def _link_or_copy(src: Path, dst: Path) -> None:
"""Hard-link ``src`` to ``dst``, falling back to a content copy."""
try:
os.link(src, dst)
except OSError:
shutil.copy2(src, dst, follow_symlinks=True)
def _stage_dir(src: Path, dst: Path, root: Path, seen: frozenset[Path]) -> None:
dst.mkdir(parents=True, exist_ok=True)
for entry in os.scandir(src):
entry_path = Path(entry.path)
dest_path = dst / entry.name
if entry.is_symlink():
target = Path(os.path.realpath(entry_path))
if not _is_within(target, root):
logger.warning("staging: dropping out-of-tree symlink %s -> %s", entry_path, target)
continue
if not target.exists():
logger.warning("staging: dropping dangling symlink %s", entry_path)
continue
if target in seen:
logger.warning("staging: dropping cyclic symlink %s -> %s", entry_path, target)
continue
if target.is_dir():
_stage_dir(target, dest_path, root, seen | {target})
else:
_link_or_copy(target, dest_path)
elif entry.is_dir(follow_symlinks=False):
_stage_dir(entry_path, dest_path, root, seen)
elif entry.is_file(follow_symlinks=False):
_link_or_copy(entry_path, dest_path)
else:
# Sockets, FIFOs, devices — not part of a source tree; skip.
logger.debug("staging: skipping non-regular entry %s", entry_path)
def stage_symlink_safe_dir(src_root: Path) -> tuple[Path, Path | None]:
"""Return ``(upload_path, staged_temp)`` for uploading ``src_root``.
``upload_path`` is safe to hand to ``LocalDir``. When the tree contains no
symlinks it is ``src_root`` itself and ``staged_temp`` is ``None``.
Otherwise a symlink-safe copy is materialized in a temp directory and both
returned values point at it; the caller owns removing ``staged_temp`` once
the upload completes.
"""
root = src_root.resolve()
if not tree_has_symlink(root):
return root, None
staged = Path(tempfile.mkdtemp(prefix=_STAGING_PREFIX))
try:
_stage_dir(root, staged, root, frozenset({root}))
except OSError:
shutil.rmtree(staged, ignore_errors=True)
raise
logger.info("staging: materialized symlink-safe copy of %s at %s", root, staged)
return staged, staged
+24 -11
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import logging
import shutil
from pathlib import Path
from typing import Any
@@ -12,6 +13,7 @@ from agents.sandbox.manifest import Environment, Manifest
from strix.config import load_settings
from strix.runtime.backends import get_backend
from strix.runtime.caido_bootstrap import bootstrap_caido
from strix.runtime.local_dir_staging import stage_symlink_safe_dir
logger = logging.getLogger(__name__)
@@ -29,16 +31,20 @@ _WORKSPACE_ROOT = "/workspace"
def build_session_entries(
local_sources: list[dict[str, Any]],
) -> tuple[dict[str | Path, BaseEntry], list[dict[str, Any]]]:
) -> tuple[dict[str | Path, BaseEntry], list[dict[str, Any]], list[Path]]:
"""Split local sources into copied manifest entries and host bind mounts.
Sources flagged ``mount`` are bind-mounted read-only at
``/workspace/<workspace_subdir>`` (not added to the manifest, so the SDK
does not stream them in file-by-file). Every other source becomes a
``LocalDir`` entry copied into the container as before.
``LocalDir`` entry copied into the container as before. Trees containing
symlinks (which the SDK's ``LocalDir`` walker refuses outright) are first
staged into a symlink-safe temp copy; those temp dirs are returned so the
caller can remove them once the upload completes.
"""
entries: dict[str | Path, BaseEntry] = {}
bind_mounts: list[dict[str, Any]] = []
staged_dirs: list[Path] = []
for src in local_sources:
ws_subdir = src.get("workspace_subdir") or ""
host_path = src.get("source_path") or ""
@@ -54,8 +60,11 @@ def build_session_entries(
}
)
else:
entries[ws_subdir] = LocalDir(src=resolved)
return entries, bind_mounts
upload_path, staged = stage_symlink_safe_dir(resolved)
if staged is not None:
staged_dirs.append(staged)
entries[ws_subdir] = LocalDir(src=upload_path)
return entries, bind_mounts, staged_dirs
async def create_or_reuse(
@@ -75,7 +84,7 @@ async def create_or_reuse(
logger.info("Reusing existing sandbox session for scan %s", scan_id)
return cached
entries, bind_mounts = build_session_entries(local_sources)
entries, bind_mounts, staged_dirs = build_session_entries(local_sources)
# Caido runs as an in-container sidecar; HTTP(S) traffic from any
# process started via ``session.exec`` (the SDK's Shell tool, etc.)
@@ -106,12 +115,16 @@ async def create_or_reuse(
backend_name,
image,
)
client, session = await backend(
image=image,
manifest=manifest,
exposed_ports=(_CONTAINER_CAIDO_PORT,),
bind_mounts=bind_mounts,
)
try:
client, session = await backend(
image=image,
manifest=manifest,
exposed_ports=(_CONTAINER_CAIDO_PORT,),
bind_mounts=bind_mounts,
)
finally:
for staged in staged_dirs:
shutil.rmtree(staged, ignore_errors=True)
caido_endpoint = await session.resolve_exposed_port(_CONTAINER_CAIDO_PORT)
scheme = "https" if caido_endpoint.tls else "http"
+193
View File
@@ -0,0 +1,193 @@
"""Tests for transient sandbox start retries."""
from __future__ import annotations
import shutil
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
from agents.sandbox.errors import (
LocalDirReadError,
WorkspaceArchiveWriteError,
WorkspaceStartError,
)
from strix.runtime import session_manager
from strix.runtime.backends import start_session_with_retry
class _FakeSession:
def __init__(self, failures: list[BaseException]) -> None:
self._failures = iter(failures)
async def start(self) -> None:
try:
raise next(self._failures)
except StopIteration:
return
async def resolve_exposed_port(self, _port: int) -> SimpleNamespace:
return SimpleNamespace(tls=False, host="127.0.0.1", port=48080)
class _FakeClient:
def __init__(self) -> None:
self.created = 0
self.deleted: list[_FakeSession] = []
async def create(self) -> _FakeSession:
self.created += 1
failures: list[BaseException] = []
if self.created == 1:
failures = [
WorkspaceStartError(
path=Path("/workspace"),
cause=WorkspaceArchiveWriteError(
path=Path("/workspace"),
cause=TimeoutError("transient transport timeout"),
),
)
]
return _FakeSession(failures)
async def delete(self, session: _FakeSession) -> None:
self.deleted.append(session)
async def test_transient_workspace_failure_retries_and_tears_down(
monkeypatch: pytest.MonkeyPatch,
) -> None:
client = _FakeClient()
sleeps: list[float] = []
async def record_sleep(delay: float) -> None:
sleeps.append(delay)
monkeypatch.setattr("strix.runtime.backends.asyncio.sleep", record_sleep)
session = await start_session_with_retry(client, client.create, attempts=3)
assert isinstance(session, _FakeSession)
assert client.created == 2
assert len(client.deleted) == 1
assert sleeps == [2.0]
async def test_non_transient_workspace_failure_does_not_retry() -> None:
client = _FakeClient()
session = _FakeSession([LocalDirReadError(src=Path("/workspace/repo"))])
async def create_session() -> _FakeSession:
client.created += 1
return session
with pytest.raises(LocalDirReadError):
await start_session_with_retry(client, create_session, attempts=3)
assert client.created == 1
assert client.deleted == [session]
async def test_each_transient_attempt_is_torn_down(monkeypatch: pytest.MonkeyPatch) -> None:
client = _FakeClient()
client.created = 0
sessions: list[_FakeSession] = []
sleeps: list[float] = []
async def record_sleep(delay: float) -> None:
sleeps.append(delay)
monkeypatch.setattr("strix.runtime.backends.asyncio.sleep", record_sleep)
async def create_session() -> _FakeSession:
client.created += 1
failures: list[BaseException] = []
if client.created < 3:
failures = [
WorkspaceStartError(
path=Path("/workspace"),
cause=TimeoutError("transient transport timeout"),
)
]
session = _FakeSession(failures)
sessions.append(session)
return session
result = await start_session_with_retry(client, create_session, attempts=3)
assert result is sessions[2]
assert client.deleted == sessions[:2]
assert sleeps == [2.0, 4.0]
async def test_staged_dirs_survive_retries_and_cleanup_once(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
repo = tmp_path / "repo"
repo.mkdir()
(repo / "real.txt").write_text("content")
(repo / "link.txt").symlink_to(repo / "real.txt")
client = _FakeClient()
observed_paths: list[Path] = []
sleeps: list[float] = []
original_rmtree = shutil.rmtree # pyright: ignore[reportDeprecated]
removed_paths: list[Path] = []
async def record_sleep(delay: float) -> None:
sleeps.append(delay)
monkeypatch.setattr("strix.runtime.backends.asyncio.sleep", record_sleep)
def record_rmtree(path: str | Path, **kwargs: Any) -> None:
removed_paths.append(Path(path))
original_rmtree(path, **kwargs) # pyright: ignore[reportDeprecated]
monkeypatch.setattr("strix.runtime.session_manager.shutil.rmtree", record_rmtree)
monkeypatch.setattr(
session_manager,
"load_settings",
lambda: SimpleNamespace(runtime=SimpleNamespace(backend="fake")),
)
monkeypatch.setattr(session_manager, "bootstrap_caido", _bootstrap_caido)
async def fake_backend(**kwargs: Any) -> tuple[_FakeClient, _FakeSession]:
staged_path = kwargs["manifest"].entries["repo"].src
async def create_session() -> _FakeSession:
observed_paths.append(Path(staged_path))
return await client.create()
session = await start_session_with_retry(client, create_session, attempts=3)
return client, session
def fake_get_backend(_name: str) -> Any:
return fake_backend
monkeypatch.setattr(session_manager, "get_backend", fake_get_backend)
try:
await session_manager.create_or_reuse(
"retry-test",
image="test-image",
local_sources=[
{
"source_path": str(repo),
"workspace_subdir": "repo",
}
],
)
finally:
await session_manager.cleanup("retry-test")
assert len(observed_paths) == 2
assert observed_paths[0] == observed_paths[1]
assert observed_paths[0] in removed_paths
assert not observed_paths[0].exists()
async def _bootstrap_caido(*_args: Any, **_kwargs: Any) -> object:
return object()
+108
View File
@@ -0,0 +1,108 @@
"""Tests for symlink-safe LocalDir staging."""
from __future__ import annotations
from typing import TYPE_CHECKING
from strix.runtime.local_dir_staging import stage_symlink_safe_dir, tree_has_symlink
if TYPE_CHECKING:
from pathlib import Path
def _make_repo(tmp_path: Path) -> Path:
repo = tmp_path / "repo"
(repo / "pkg").mkdir(parents=True)
(repo / "pkg" / "mod.py").write_text("x = 1\n")
(repo / "README.md").write_text("readme\n")
return repo
def test_tree_without_symlinks_used_as_is(tmp_path: Path) -> None:
repo = _make_repo(tmp_path)
upload_path, staged = stage_symlink_safe_dir(repo)
assert staged is None
assert upload_path == repo.resolve()
assert not tree_has_symlink(repo)
def test_in_tree_file_symlink_is_dereferenced(tmp_path: Path) -> None:
repo = _make_repo(tmp_path)
(repo / "link.py").symlink_to(repo / "pkg" / "mod.py")
upload_path, staged = stage_symlink_safe_dir(repo)
assert staged is not None
assert upload_path == staged
assert not (staged / "link.py").is_symlink()
assert (staged / "link.py").read_text() == "x = 1\n"
assert (staged / "pkg" / "mod.py").read_text() == "x = 1\n"
assert not tree_has_symlink(staged)
def test_in_tree_relative_dir_symlink_is_dereferenced(tmp_path: Path) -> None:
repo = _make_repo(tmp_path)
(repo / "pkg_alias").symlink_to("pkg")
_upload, staged = stage_symlink_safe_dir(repo)
assert staged is not None
assert (staged / "pkg_alias" / "mod.py").read_text() == "x = 1\n"
assert not tree_has_symlink(staged)
def test_out_of_tree_symlink_is_dropped(tmp_path: Path) -> None:
repo = _make_repo(tmp_path)
outside = tmp_path / "outside.txt"
outside.write_text("secret\n")
(repo / "escape.txt").symlink_to(outside)
(repo / "abs_escape").symlink_to("/etc")
_upload, staged = stage_symlink_safe_dir(repo)
assert staged is not None
assert not (staged / "escape.txt").exists()
assert not (staged / "abs_escape").exists()
assert (staged / "README.md").exists()
def test_dangling_symlink_is_dropped(tmp_path: Path) -> None:
repo = _make_repo(tmp_path)
(repo / "dangling").symlink_to(repo / "does-not-exist")
_upload, staged = stage_symlink_safe_dir(repo)
assert staged is not None
assert not (staged / "dangling").exists()
assert not (staged / "dangling").is_symlink()
def test_cyclic_symlink_terminates(tmp_path: Path) -> None:
repo = _make_repo(tmp_path)
(repo / "self").symlink_to(repo)
(repo / "pkg" / "up").symlink_to("..")
_upload, staged = stage_symlink_safe_dir(repo)
assert staged is not None
assert (staged / "README.md").exists()
assert not tree_has_symlink(staged)
def test_nested_symlinks_inside_linked_dir(tmp_path: Path) -> None:
repo = _make_repo(tmp_path)
shared = repo / "shared"
shared.mkdir()
(shared / "conf.json").write_text("{}\n")
(shared / "escape").symlink_to("/etc/passwd")
(repo / "pkg" / "shared_link").symlink_to(shared)
_upload, staged = stage_symlink_safe_dir(repo)
assert staged is not None
assert (staged / "pkg" / "shared_link" / "conf.json").read_text() == "{}\n"
assert not (staged / "pkg" / "shared_link" / "escape").exists()
assert not (staged / "shared" / "escape").exists()
+24 -4
View File
@@ -18,15 +18,18 @@ def _source(subdir: str, path: str, *, mount: bool = False) -> dict[str, Any]:
def test_copied_source_becomes_localdir_entry(tmp_path: Path) -> None:
entries, bind_mounts = build_session_entries([_source("repo", str(tmp_path))])
entries, bind_mounts, staged_dirs = build_session_entries([_source("repo", str(tmp_path))])
assert bind_mounts == []
assert staged_dirs == []
assert isinstance(entries["repo"], LocalDir)
assert entries["repo"].src == tmp_path.resolve()
def test_mounted_source_becomes_bind_mount(tmp_path: Path) -> None:
entries, bind_mounts = build_session_entries([_source("repo", str(tmp_path), mount=True)])
entries, bind_mounts, _staged = build_session_entries(
[_source("repo", str(tmp_path), mount=True)]
)
assert entries == {}
assert bind_mounts == [
@@ -44,7 +47,7 @@ def test_mixed_sources_split_correctly(tmp_path: Path) -> None:
copied.mkdir()
mounted.mkdir()
entries, bind_mounts = build_session_entries(
entries, bind_mounts, _staged = build_session_entries(
[
_source("copied", str(copied)),
_source("mounted", str(mounted), mount=True),
@@ -57,7 +60,7 @@ def test_mixed_sources_split_correctly(tmp_path: Path) -> None:
def test_incomplete_sources_are_skipped() -> None:
entries, bind_mounts = build_session_entries(
entries, bind_mounts, staged_dirs = build_session_entries(
[
{"source_path": "", "workspace_subdir": "x"},
{"source_path": "/p", "workspace_subdir": ""},
@@ -65,3 +68,20 @@ def test_incomplete_sources_are_skipped() -> None:
)
assert entries == {}
assert bind_mounts == []
assert staged_dirs == []
def test_symlink_tree_is_staged(tmp_path: Path) -> None:
repo = tmp_path / "repo"
repo.mkdir()
(repo / "real.txt").write_text("content")
(repo / "link.txt").symlink_to(repo / "real.txt")
entries, _mounts, staged_dirs = build_session_entries([_source("repo", str(repo))])
assert len(staged_dirs) == 1
entry = entries["repo"]
assert isinstance(entry, LocalDir)
assert entry.src == staged_dirs[0]
assert not (staged_dirs[0] / "link.txt").is_symlink()
assert (staged_dirs[0] / "link.txt").read_text() == "content"