From 03857c9becebbe04b04a8e256d68af7545d8d007 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:00:25 +0200 Subject: [PATCH] search-with-proof: root-no-userns isolation mode for gVisor The kube-ax probe showed bwrap cannot map uids in gVisor as an unprivileged user. As root (gVisor is the outer boundary) the sandbox uses pid/ipc/uts/net/mount namespaces without a user namespace and drops the artefact to its own uid via setpriv; the mode is recorded. The probe reports each namespace kind separately. Co-Authored-By: Claude Opus 5.5 --- plane/Dockerfile.search-task | 10 +- src/openultrasast/search/_sandbox.py | 59 ++++++++-- src/openultrasast/search/oracles.py | 20 +++- src/openultrasast/search/probe.py | 27 ++++- src/openultrasast/search/verify.py | 33 +++++- src/openultrasast/search/worker.py | 17 ++- tests/test_search_image.py | 3 + tests/test_search_isolation.py | 160 +++++++++++++++++++++++++++ tests/test_search_worker.py | 3 + 9 files changed, 308 insertions(+), 24 deletions(-) create mode 100644 tests/test_search_isolation.py diff --git a/plane/Dockerfile.search-task b/plane/Dockerfile.search-task index 972a381..5cab3ac 100644 --- a/plane/Dockerfile.search-task +++ b/plane/Dockerfile.search-task @@ -23,8 +23,10 @@ RUN python3 -m venv /venv \ && /venv/bin/pip install --no-cache-dir . \ && python3 --version && node --version && tsc --version && php --version \ && javac -version && sqlite3 --version && bwrap --version \ - && prlimit --version && nsenter --version && chromium --version -# Browser sandbox requires a non-root caller; the AX runner needs no root privileges. -RUN useradd --create-home --uid 10001 search -USER search + && setpriv --version && prlimit --version && nsenter --version && chromium --version +# Dedicated identity for artefacts, apps and Chromium; the verifier keeps root. +RUN groupadd --gid 10001 search \ + && useradd --create-home --uid 10001 --gid 10001 search +# gVisor is the outer boundary; inner isolation needs root without user namespaces. +USER root # Do not replace the AX runner's ENTRYPOINT or CMD. diff --git a/src/openultrasast/search/_sandbox.py b/src/openultrasast/search/_sandbox.py index 8ea11b1..c6feb1f 100644 --- a/src/openultrasast/search/_sandbox.py +++ b/src/openultrasast/search/_sandbox.py @@ -21,6 +21,43 @@ from pathlib import Path from ..sandbox import SandboxJob, SandboxResult +ARTEFACT_UID = 10001 +ARTEFACT_GID = 10001 + + +def isolation_mode() -> str: + """Root containers need no user namespace; ordinary host callers retain it.""" + return "root-no-userns" if os.geteuid() == 0 else "userns" + + +def drop_privileges() -> list[str]: + return [ + "/usr/bin/setpriv", + f"--reuid={ARTEFACT_UID}", + f"--regid={ARTEFACT_GID}", + "--clear-groups", + "--no-new-privs", + "--bounding-set=-all", + "--inh-caps=-all", + "--ambient-caps=-all", + "--", + ] + + +def capability_args() -> list[str]: + # Retain only what the trusted setpriv trampoline needs, then erase the + # bounding/inheritable/ambient sets before executing any untrusted code. + args = ["--cap-drop", "ALL"] + if isolation_mode() == "root-no-userns": + for capability in ("CAP_SETUID", "CAP_SETGID", "CAP_SETPCAP"): + args += ["--cap-add", capability] + return args + + +def writable_directory(path: Path) -> None: + if isolation_mode() == "root-no-userns": + os.chown(path, ARTEFACT_UID, ARTEFACT_GID, follow_symlinks=False) + class IsolationUnavailable(RuntimeError): """The isolation instrument failed, so no product outcome can be recorded.""" @@ -31,6 +68,8 @@ def isolation_check(*, timeout_seconds: int = 5) -> None: try: with tempfile.TemporaryDirectory(prefix="isolation-check-") as directory: root = Path(directory) + if isolation_mode() == "root-no-userns": + root.chmod(0o755) (root / "probe").write_text("isolation-ready\n") scratch = root / "scratch" scratch.mkdir() @@ -78,28 +117,33 @@ def run( resource.setrlimit(resource.RLIMIT_CPU, (job.timeout_seconds + 1,) * 2) argv = ["bwrap", "--die-with-parent", "--new-session", "--unshare-pid", "--unshare-ipc", "--unshare-uts"] + mode = isolation_mode() if namespace_pid is None: - argv += ["--unshare-user"] + if mode == "userns": + argv += ["--unshare-user"] else: argv = [ "/usr/bin/nsenter", - "--preserve-credentials", - f"--user=/proc/{namespace_pid}/ns/user", + *(["--preserve-credentials", f"--user=/proc/{namespace_pid}/ns/user"] if mode == "userns" else []), f"--net=/proc/{namespace_pid}/ns/net", "--", *argv, ] - argv += ["--cap-drop", "ALL", "--clearenv"] + argv += [*capability_args(), "--clearenv"] for path in ("/usr", "/lib", "/lib64", "/bin"): if Path(path).exists(): argv += ["--ro-bind", path, path] argv += ["--ro-bind", str(job.repo_root), "/workspace"] if scratch_bytes is None: + writable_directory(scratch) argv += ["--bind", str(scratch), "/scratch"] else: if scratch_bytes < 1: raise ValueError("positive scratch limit required") - argv += ["--size", str(scratch_bytes), "--tmpfs", "/scratch"] + argv += ["--size", str(scratch_bytes)] + if mode == "root-no-userns": + argv += ["--perms", "0777"] + argv += ["--tmpfs", "/scratch"] for target, source in (mounts or {}).items(): argv += ["--ro-bind", str(source), target] argv += [ @@ -122,12 +166,13 @@ def run( with tempfile.TemporaryFile() as policy, tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err: policy.write(_filter(namespace_pid is not None)) policy.seek(0) - # Set NPROC after entering the user namespace (per-namespace since Linux 5.14), - # so the host uid's existing processes cannot prevent bwrap from starting. + # Apply NPROC after namespace setup and any uid drop. In root mode the + # dedicated uid shares this process budget across concurrent sandboxes. argv += [ "--seccomp", str(policy.fileno()), "--", + *(drop_privileges() if mode == "root-no-userns" else []), "/usr/bin/prlimit", f"--nproc={job.pids_limit}:{job.pids_limit}", "--", diff --git a/src/openultrasast/search/oracles.py b/src/openultrasast/search/oracles.py index ee2566a..046f70d 100644 --- a/src/openultrasast/search/oracles.py +++ b/src/openultrasast/search/oracles.py @@ -22,6 +22,9 @@ from pathlib import Path from typing import Protocol from urllib.parse import quote +if __name__ != "__main__": + from . import _sandbox + @dataclass(frozen=True) class Canary: @@ -30,7 +33,7 @@ class Canary: @classmethod def fresh(cls, root: Path) -> Canary: - root.mkdir() + root.mkdir(mode=0o700) return cls(root, secrets.token_hex(24)) @@ -66,6 +69,7 @@ class PathOracle(OutputOracle): super().prepare(canary) (canary.root / "public").mkdir() (canary.root / "canary").write_text(canary.nonce) + (canary.root / "canary").chmod(0o600) return {"SERVED_ROOT": "/fixture/public"} @@ -75,6 +79,7 @@ class SQLOracle(OutputOracle): with sqlite3.connect(canary.root / "fixture.db") as db: db.execute("CREATE TABLE records (name TEXT, value TEXT, public INTEGER)") db.executemany("INSERT INTO records VALUES (?, ?, ?)", [("ordinary", "public value", 1), ("secret", canary.nonce, 0)]) + (canary.root / "fixture.db").chmod(0o600) return {"DATABASE": "/fixture/fixture.db"} @@ -100,8 +105,7 @@ class SSRFOracle(OutputOracle): self.process = subprocess.Popen( [ "/usr/bin/unshare", - "--user", - "--map-root-user", + *(["--user", "--map-root-user"] if _sandbox.isolation_mode() == "userns" else []), "--net", "/usr/bin/python3", "-I", @@ -145,7 +149,8 @@ class BrowserExecutor: Payloads call alert("ousast-xss"). Reflection/escaped HTML cannot set the fresh DOM marker. A restrictive CSP disables external resources and connections; - Chromium's sandbox stays enabled and each capture uses a fresh profile. + Chromium's sandbox stays enabled for userns; root containers use the outer + boundary and drop the browser uid. Each capture uses a fresh profile. Launch failures are unavailable execution, never a negative observation. """ @@ -155,6 +160,7 @@ class BrowserExecutor: if not selected: raise OSError("headless chromium unavailable") self.binary: str = selected + self.no_sandbox = _sandbox.isolation_mode() == "root-no-userns" def __call__(self, document: str, nonce: str) -> bool: marker = secrets.token_hex(24) @@ -175,7 +181,9 @@ class BrowserExecutor: with tempfile.TemporaryDirectory(prefix="ousast-browser-") as profile: # HTML can navigate even with CSP. A private network namespace, not # browser flags, is the no-egress boundary. Expose runtime only. - sandbox = ["bwrap", "--unshare-all", "--die-with-parent", "--new-session"] + _sandbox.writable_directory(Path(profile)) + namespaces = ["--unshare-pid", "--unshare-ipc", "--unshare-uts", "--unshare-net"] if self.no_sandbox else ["--unshare-all"] + sandbox = ["bwrap", *namespaces, "--die-with-parent", "--new-session", *_sandbox.capability_args()] for path in ("/usr", "/bin", "/lib", "/lib64"): if Path(path).exists(): sandbox += ["--ro-bind", path, path] @@ -198,7 +206,9 @@ class BrowserExecutor: done = subprocess.run( [ *sandbox, + *(_sandbox.drop_privileges() if self.no_sandbox else []), self.binary, + *(["--no-sandbox"] if self.no_sandbox else []), "--headless", "--dump-dom", "--disable-gpu", diff --git a/src/openultrasast/search/probe.py b/src/openultrasast/search/probe.py index 5e50148..16c7d05 100644 --- a/src/openultrasast/search/probe.py +++ b/src/openultrasast/search/probe.py @@ -84,8 +84,33 @@ def image_facts() -> dict[str, Any]: return facts +def namespace_facts() -> dict[str, Any]: + """Try capabilities independently, even when the combined preflight fails.""" + commands = { + name: ["bwrap", f"--unshare-{name}", "--bind", "/", "/", "--", "/bin/true"] + for name in ("user", "pid", "net", "ipc", "uts", "cgroup") + } + commands["mount"] = ["bwrap", "--bind", "/", "/", "--", "/bin/true"] + commands["setpriv"] = [*_sandbox.drop_privileges(), "/usr/bin/id"] + facts = {} + for name, argv in commands.items(): + try: + done = subprocess.run(argv, capture_output=True, text=True, timeout=5, env={"PATH": os.defpath}) + facts[name] = {"exit_code": done.returncode, "stderr": done.stderr[:2000], "stdout": done.stdout[:2000]} + except (OSError, subprocess.SubprocessError) as exc: + facts[name] = {"exit_code": None, "stderr": str(exc)} + return facts + + def probe(root: Path) -> dict[str, Any]: - record: dict[str, Any] = {"status": "ok", "oracles": {}, "image": image_facts(), "isolation_check": None} + record: dict[str, Any] = { + "status": "ok", + "oracles": {}, + "image": image_facts(), + "namespaces": namespace_facts(), + "isolation_check": None, + "isolation_mode": _sandbox.isolation_mode(), + } start = time.monotonic() try: _sandbox.isolation_check() diff --git a/src/openultrasast/search/verify.py b/src/openultrasast/search/verify.py index f51aa5d..3b1f200 100644 --- a/src/openultrasast/search/verify.py +++ b/src/openultrasast/search/verify.py @@ -20,7 +20,7 @@ import stat import tempfile import time from collections.abc import Callable -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any @@ -61,6 +61,8 @@ class SideRecord: ready_seconds: float = 0.0 run_seconds: float = 0.0 scratch_peak_bytes: int = 0 + isolation_mode: str = field(default_factory=_sandbox.isolation_mode) + chromium_no_sandbox: bool = False @dataclass(frozen=True) @@ -69,6 +71,8 @@ class VerificationRecord: sides: tuple[SideRecord, ...] elapsed_seconds: float reason: str + isolation_mode: str = field(default_factory=_sandbox.isolation_mode) + chromium_no_sandbox: bool = False def _tree(root: Path) -> str: @@ -125,7 +129,15 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe def record(outcome: str, reason: str) -> SideRecord: return SideRecord( - outcome, tuple(observations), time.monotonic() - started, reason, timings["build"], timings["ready"], timings["run"], disk_peak + outcome, + tuple(observations), + time.monotonic() - started, + reason, + timings["build"], + timings["ready"], + timings["run"], + disk_peak, + chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)), ) try: @@ -170,7 +182,14 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe app_env = oracle.prepare(canary) app_scratch = canary.root if family == "command" else root / "app" app_scratch.mkdir(exist_ok=True) - app_mounts = {**mounts, "/fixture": canary.root} + fixture = canary.root + if _sandbox.isolation_mode() == "root-no-userns" and family != "command": + # Observer originals remain private. Only the app gets a + # read-only copy of the fixtures its interface requires. + fixture = shutil.copytree(canary.root, root / "app-fixture") + for path in [fixture, *fixture.rglob("*")]: + path.chmod(0o555 if path.is_dir() else 0o444) + app_mounts = {**mounts, "/fixture": fixture} ready = timed( "ready", _sandbox.run, @@ -262,7 +281,13 @@ def verify( outcome, reason = "demonstrated", "owned effect in 3/3 affected runs and 0/3 safe runs" else: outcome, reason = "inconclusive", "no consistent differential effect" - return VerificationRecord(outcome, sides, time.monotonic() - started, reason) + return VerificationRecord( + outcome, + sides, + time.monotonic() - started, + reason, + chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)), + ) def main() -> None: diff --git a/src/openultrasast/search/worker.py b/src/openultrasast/search/worker.py index 94d0348..66ab30a 100644 --- a/src/openultrasast/search/worker.py +++ b/src/openultrasast/search/worker.py @@ -251,6 +251,7 @@ class Worker: scratch_bytes=int(self.limits["disk_bytes"]), ) return { + "isolation_mode": _sandbox.isolation_mode(), "exit_code": result.exit_code, "stdout": result.stdout[:MAX_OUTPUT], "stderr": result.stderr[:MAX_OUTPUT], @@ -362,12 +363,22 @@ class Worker: result = self.explore(task, meter) else: raise ValueError("verify requires the trusted verifier executor") - return {"status": "ok", **result, "cost_usd": meter.spent} + return {"status": "ok", **result, "cost_usd": meter.spent, "isolation_mode": _sandbox.isolation_mode()} except _sandbox.IsolationUnavailable: - return {"status": "instrument_failure", "failure": "sandbox isolation unavailable", "cost_usd": meter.spent} + return { + "status": "instrument_failure", + "failure": "sandbox isolation unavailable", + "cost_usd": meter.spent, + "isolation_mode": _sandbox.isolation_mode(), + } except Exception as exc: # Provider exceptions can contain keys/URLs. Never persist their text. - return {"status": "execution_failure", "failure": type(exc).__name__, "cost_usd": meter.spent} + return { + "status": "execution_failure", + "failure": type(exc).__name__, + "cost_usd": meter.spent, + "isolation_mode": _sandbox.isolation_mode(), + } def main() -> None: diff --git a/tests/test_search_image.py b/tests/test_search_image.py index f04a694..b1807f6 100644 --- a/tests/test_search_image.py +++ b/tests/test_search_image.py @@ -10,6 +10,9 @@ def test_image_matches_runner_and_publishes_digest(): assert runner in engine and runner in image for package in ("python3", "nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"): assert package in image + assert image.rsplit("USER ", 1)[1].startswith("root\n") + assert "--uid 10001 --gid 10001 search" in image + assert "setpriv --version" in image assert "17-jdk" in image and "--no-install-recommends" in image assert "rm -rf /var/lib/apt/lists/" in image workflow_text = Path(".github/workflows/engine-image.yml").read_text() diff --git a/tests/test_search_isolation.py b/tests/test_search_isolation.py new file mode 100644 index 0000000..e2b8fd3 --- /dev/null +++ b/tests/test_search_isolation.py @@ -0,0 +1,160 @@ +"""Root command construction is tested without requiring root privileges.""" + +import subprocess +from dataclasses import asdict + +import pytest + +from openultrasast.sandbox import SandboxJob +from openultrasast.search import _sandbox, probe +from openultrasast.search.oracles import BrowserExecutor, Canary, SSRFOracle +from openultrasast.search.verify import SideRecord, VerificationRecord + + +@pytest.mark.parametrize("uid,mode", [(0, "root-no-userns"), (1000, "userns")]) +def test_selection_and_records(monkeypatch, uid, mode): + monkeypatch.setattr(_sandbox.os, "geteuid", lambda: uid) + assert _sandbox.isolation_mode() == mode + assert asdict(SideRecord("observed", (), 0, ""))["isolation_mode"] == mode + assert asdict(VerificationRecord("inconclusive", (), 0, ""))["isolation_mode"] == mode + + +@pytest.mark.parametrize("pid", [None, 123]) +def test_root_argv(monkeypatch, tmp_path, pid): + monkeypatch.setattr(_sandbox.os, "geteuid", lambda: 0) + monkeypatch.setattr(_sandbox.os, "chown", lambda *a, **kw: None) + monkeypatch.setattr(_sandbox.os, "killpg", lambda *a: None) + commands = [] + + class Process: + returncode = 0 + pid = 987 + + def __init__(self, argv, **kwargs): + commands.append(argv) + assert kwargs["pass_fds"] + assert kwargs["preexec_fn"] + + def wait(self, **kwargs): + return 0 + + monkeypatch.setattr(_sandbox.subprocess, "Popen", Process) + job = SandboxJob("", ("/bin/true",), tmp_path, {}, 5, 256, 128) + _sandbox.run(job, scratch=tmp_path, namespace_pid=pid) + argv = commands[0] + assert "--unshare-user" not in argv + assert not any(a.startswith("--user=") for a in argv) + assert {"--unshare-pid", "--unshare-ipc", "--unshare-uts", "--seccomp"} <= set(argv) + drop = _sandbox.drop_privileges() + start = argv.index(drop[0]) + assert argv[start : start + len(drop)] == drop + assert argv.index("/usr/bin/prlimit") > start + assert "--nproc=128:128" in argv + if pid: + assert "--net=/proc/123/ns/net" in argv + else: + assert "/usr/bin/nsenter" not in argv + + +def test_root_browser(monkeypatch): + monkeypatch.setattr(_sandbox.os, "geteuid", lambda: 0) + monkeypatch.setattr(_sandbox.os, "chown", lambda *a, **kw: None) + + def run(argv, **kwargs): + assert "--unshare-user" not in argv and "--unshare-all" not in argv + assert "--unshare-net" in argv and "--unshare-pid" in argv + assert "--no-sandbox" in argv + assert argv.index("/usr/bin/setpriv") < argv.index("/fake/chromium") + return subprocess.CompletedProcess(argv, 0, "", "") + + monkeypatch.setattr(subprocess, "run", run) + browser = BrowserExecutor("/fake/chromium") + assert browser.no_sandbox + assert not browser("", "nonce") + + +def test_namespace_diagnostics_precede_failed_preflight(monkeypatch, tmp_path): + calls = [] + + def run(argv, **kwargs): + calls.append(argv) + return subprocess.CompletedProcess(argv, 1, "", "denied") + + def fail(): + assert len(calls) == 8 + raise _sandbox.IsolationUnavailable("denied") + + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(probe, "image_facts", lambda: {}) + monkeypatch.setattr(_sandbox, "isolation_check", fail) + record = probe.probe(tmp_path) + assert record["status"] == "instrument_failure" + assert set(record["namespaces"]) == {"user", "pid", "net", "ipc", "uts", "cgroup", "mount", "setpriv"} + assert all(row["exit_code"] == 1 and row["stderr"] == "denied" for row in record["namespaces"].values()) + assert record["isolation_mode"] == _sandbox.isolation_mode() + + +@pytest.mark.parametrize("uid", [0, 1000]) +def test_listener_namespace_setup(monkeypatch, tmp_path, uid): + import os + + monkeypatch.setattr(_sandbox.os, "geteuid", lambda: uid) + read_fd, write_fd = os.pipe() + os.write(write_fd, b"[123, 4567]\n") + os.close(write_fd) + stream = os.fdopen(read_fd) + commands = [] + + class Process: + stdout = stream + + def __init__(self, argv, **kwargs): + commands.append(argv) + + def terminate(self): + pass + + def wait(self, **kwargs): + return 0 + + monkeypatch.setattr(subprocess, "Popen", Process) + oracle = SSRFOracle() + try: + env = oracle.prepare(Canary.fresh(tmp_path / "fixture")) + assert oracle.namespace_pid == 123 + assert ":4567/" in env["CALLBACK_URL"] + assert "--net" in commands[0] + assert ("--user" in commands[0]) == (uid != 0) + assert ("--map-root-user" in commands[0]) == (uid != 0) + finally: + oracle.close() + + +def test_root_fixture_permissions_and_browser_records(monkeypatch, tmp_path): + from openultrasast.sandbox import SandboxResult + from openultrasast.search import verify as verifier + + monkeypatch.setattr(_sandbox.os, "geteuid", lambda: 0) + monkeypatch.setattr(_sandbox, "isolation_check", lambda **kw: None) + sides, demos = probe.materialise(tmp_path / "pair", "path") + app_calls = [] + + def run(job, *, scratch, mounts, **kwargs): + if "/fixture" in mounts: + fixture = mounts["/fixture"] + assert fixture.name == "app-fixture" + original = fixture.parent / "fixture" + assert original.stat().st_mode & 0o777 == 0o700 + assert (original / "canary").stat().st_mode & 0o777 == 0o600 + assert (fixture / "canary").read_bytes() == (original / "canary").read_bytes() + assert fixture.stat().st_mode & 0o777 == 0o555 + app_calls.append(job) + return SandboxResult(0, "", "", False) + + monkeypatch.setattr(_sandbox, "run", run) + browser = BrowserExecutor("/fake/chromium") + record = verifier.verify(*sides, demos["real"], "path", browser=browser) + assert app_calls + assert record.isolation_mode == "root-no-userns" + assert record.chromium_no_sandbox + assert all(s.chromium_no_sandbox and s.isolation_mode == "root-no-userns" for s in record.sides) diff --git a/tests/test_search_worker.py b/tests/test_search_worker.py index 51009b4..b328f7a 100644 --- a/tests/test_search_worker.py +++ b/tests/test_search_worker.py @@ -4,6 +4,7 @@ from dataclasses import asdict import pytest from openultrasast.plane.memory import FileStore +from openultrasast.search import _sandbox from openultrasast.search.board import Board from openultrasast.search.budget import SearchBudget from openultrasast.search.coordinator import Coordinator, SearchTask @@ -29,6 +30,7 @@ def test_reason_no_tools_and_bounded(tmp_path): result = Worker(model, repo=tmp_path, demo=tmp_path / "demo")(task()) assert result["intents"][0]["description"] == "inspect" assert model.calls[0]["tools"] == [] + assert result["isolation_mode"] == _sandbox.isolation_mode() assert result["cost_usd"] == 0 model = StubModel([reply({"intents": [{"description": "x"}] * 4})]) assert Worker(model, repo=tmp_path, demo=tmp_path / "demo")(task())["status"] == "execution_failure" @@ -59,6 +61,7 @@ def test_run_scrubs_model_key_in_real_sandbox(tmp_path, monkeypatch): "timeout_seconds": 2, }, ) + assert result["isolation_mode"] == _sandbox.isolation_mode() assert result["exit_code"] == 0, result assert result["stdout"].splitlines()[0] == "False" assert "must-not-reach-repo" not in json.dumps(result)