"""Owned observations and adversarial artefacts, with real isolated toy processes."""
import os
import shutil
import signal
import socket
from pathlib import Path
import pytest
from openultrasast.search.verify import Side, verify
FIXTURE = Path(__file__).parents[1] / "src/openultrasast/search/probe_apps/app.py.txt"
PAYLOADS = {
"path": "../canary",
"sql": "' OR public=0 --",
"command": '; touch "$PROOF_MARKER"',
"ssrf": "$CALLBACK_URL",
}
@pytest.fixture
def pair(tmp_path):
sides = []
for name in ("vulnerable", "fixed"):
root = tmp_path / name
root.mkdir()
shutil.copyfile(FIXTURE, root / "app.py")
if name == "fixed":
(root / "fixed").touch()
sides.append(Side(root, ("/usr/bin/python3", "-I", "/workspace/app.py")))
demo = tmp_path / "demo"
demo.mkdir()
(demo / "build.sh").write_text("test -s /workspace/app.py\n")
return sides, demo
def request(demo, family):
import json
(demo / "request.json").write_text(json.dumps([family, PAYLOADS[family]]))
@pytest.mark.parametrize("family", ["path", "sql", "command", "ssrf"])
def test_real_differential(pair, family):
if family == "ssrf":
try:
with socket.socket():
pass
except PermissionError:
pytest.skip("host policy prohibits even localhost sockets")
sides, demo = pair
request(demo, family)
result = verify(*sides, demo, family)
assert result.outcome == "demonstrated", result
assert [r.observed for r in result.sides[0].runs] == [True] * 3
assert [r.observed for r in result.sides[1].runs] == [False] * 3
assert result.elapsed_seconds > 0
@pytest.mark.parametrize(
"script",
[
"cat /fixture/canary; echo success; exit 42",
"if test ! -f /workspace/fixed; then echo demonstrated; fi",
"echo hacked > /workspace/sitecustomize.py",
"echo hacked > /usr/lib/python3.12/sitecustomize.py",
"echo hacked > /deps/0/module.py",
"cat /proc/1/environ; echo demonstrated",
'/usr/bin/python3 -I -c "import socket; socket.create_connection(("127.0.0.1",12345))"',
],
)
def test_gaming_cannot_demonstrate(pair, tmp_path, script):
sides, demo = pair
deps = tmp_path / "deps"
deps.mkdir()
(deps / "module.py").write_text("original")
sides = [Side(s.checkout, s.command, dependencies=(deps,)) for s in sides]
(demo / "attack.sh").write_text(script)
result = verify(*sides, demo, "path")
assert result.outcome != "demonstrated"
assert (deps / "module.py").read_text() == "original"
assert not (sides[0].checkout / "sitecustomize.py").exists()
def test_attack_exit_and_stdout_ignored(pair):
sides, demo = pair
(demo / "attack.sh").write_text('"$TARGET" path ../canary\necho fake\nexit 99\n')
assert verify(*sides, demo, "path").outcome == "demonstrated"
def test_safe_build_failure_is_inconclusive(pair):
sides, demo = pair
request(demo, "path")
(demo / "build.sh").write_text("test ! -f /workspace/fixed")
result = verify(*sides, demo, "path")
assert result.outcome == "inconclusive"
assert result.sides[1].outcome == "could_not_build"
def test_readiness_failure(pair):
sides, demo = pair
request(demo, "path")
(sides[1].checkout / "broken").touch()
result = verify(*sides, demo, "path")
assert result.outcome == "inconclusive"
assert result.sides[1].outcome == "could_not_run"
@pytest.mark.parametrize("family", ["xss", "deserialisation", "access_control", "secrets"])
def test_no_oracle(pair, family):
sides, demo = pair
assert verify(*sides, demo, family).outcome == "no_oracle"
def test_flaky_observation_is_inconclusive(pair, monkeypatch):
from openultrasast.search import oracles
nonces = iter(["a" * 48, "b" * 48, "c" * 48] * 2)
monkeypatch.setattr(oracles.secrets, "token_hex", lambda _: next(nonces))
sides, demo = pair
(sides[0].checkout / "flaky").touch()
request(demo, "path")
result = verify(*sides, demo, "path")
assert result.outcome == "inconclusive"
assert [r.observed for r in result.sides[0].runs] == [True, False, False]
def test_environment_injection_stays_in_artefact(pair, monkeypatch):
sides, demo = pair
monkeypatch.setenv("PYTHONPATH", "/bogus")
monkeypatch.setenv("NODE_OPTIONS", "--require=/bogus")
monkeypatch.setenv("LD_PRELOAD", "/bogus")
(demo / "attack.sh").write_text(
'test -z "$PYTHONPATH$NODE_OPTIONS$LD_PRELOAD" || exit 3\n'
'echo "raise RuntimeError()" > /scratch/sitecustomize.py\n'
"export PYTHONPATH=/scratch NODE_OPTIONS=--require=/scratch/sitecustomize.py LD_PRELOAD=/bogus\n"
'"$TARGET" path ../canary\n'
)
assert verify(*sides, demo, "path").outcome == "demonstrated"
def test_untracked_file_outside_scratch_refused(pair):
sides, demo = pair
(demo / "attack.sh").write_text('echo hacked > /untracked\n"$TARGET" path ../canary\n')
assert verify(*sides, demo, "path").outcome == "inconclusive"
def test_unknown_app_failure_cannot_prove_effect(pair):
sides, demo = pair
(sides[0].checkout / "app.py").write_text('raise RuntimeError("no app")')
request(demo, "path")
assert verify(*sides, demo, "path").outcome == "could_not_run"
def test_affected_build_failure(pair):
sides, demo = pair
request(demo, "path")
(demo / "build.sh").write_text("test -f /workspace/fixed")
assert verify(*sides, demo, "path").outcome == "could_not_build"
def test_missing_isolation_fails_closed(pair, monkeypatch):
from openultrasast.search import _sandbox
def unavailable(*args, **kwargs):
raise FileNotFoundError("bwrap unavailable")
monkeypatch.setattr(_sandbox, "run", unavailable)
sides, demo = pair
request(demo, "path")
with pytest.raises(_sandbox.IsolationUnavailable, match="bwrap unavailable"):
verify(*sides, demo, "path")
def test_bwrap_failure_raises_before_product_outcomes(pair, monkeypatch):
from openultrasast.sandbox import SandboxResult
from openultrasast.search import _sandbox
calls = []
stderr = "bwrap: Creating new namespace failed: Resource temporarily unavailable"
def failed(job, **kwargs):
calls.append(job)
return SandboxResult(1, "", stderr, False)
monkeypatch.setattr(_sandbox, "run", failed)
sides, demo = pair
with pytest.raises(_sandbox.IsolationUnavailable, match=stderr):
verify(*sides, demo, "path")
assert len(calls) == 1
assert calls[0].command == ("/bin/cat", "/workspace/probe")
def test_verification_with_many_host_processes(pair, monkeypatch):
from openultrasast.search import _sandbox
checks = []
original = _sandbox.isolation_check
def checked(**kwargs):
checks.append(True)
return original(**kwargs)
monkeypatch.setattr(_sandbox, "isolation_check", checked)
sides, demo = pair
request(demo, "path")
children = []
try:
for _ in range(300):
pid = os.fork()
if pid == 0:
try:
while True:
signal.pause()
finally:
os._exit(0)
children.append(pid)
assert verify(*sides, demo, "path").outcome == "demonstrated"
assert checks == [True]
finally:
for pid in children:
os.kill(pid, signal.SIGKILL)
for pid in children:
os.waitpid(pid, 0)
def test_symlink_input_refused(pair):
sides, demo = pair
(demo / "escape").symlink_to("/etc/passwd")
request(demo, "path")
assert verify(*sides, demo, "path").outcome == "inconclusive"
def test_xss_executor_contract(tmp_path):
from openultrasast.search.oracles import Canary, oracle_for
calls = []
def browser(document, nonce):
calls.append((document, nonce))
return document == ""
oracle = oracle_for("xss", browser)
canary = Canary.fresh(tmp_path / "fixture")
assert oracle.prepare(canary) == {}
oracle.capture("")
assert oracle.observe()[0]
assert calls == [("", canary.nonce)]
def test_six_fresh_canaries(pair, monkeypatch):
from openultrasast.search import oracles
nonces = []
prepare = oracles.PathOracle.prepare
def record(self, canary):
nonces.append(canary.nonce)
return prepare(self, canary)
monkeypatch.setattr(oracles.PathOracle, "prepare", record)
sides, demo = pair
request(demo, "path")
assert verify(*sides, demo, "path").outcome == "demonstrated"
assert len(set(nonces)) == 6
def test_readiness_effect_cannot_prove_ssrf(pair, monkeypatch):
from openultrasast.search import verify as module
from openultrasast.search.oracles import OutputOracle
class ReadinessOracle(OutputOracle):
def observe(self):
return True, "request during readiness"
monkeypatch.setattr(module, "oracle_for", lambda *args: ReadinessOracle())
sides, demo = pair
result = module.verify(*sides, demo, "ssrf")
assert result.outcome == "inconclusive"
assert not result.sides[0].runs
def test_dependency_hash_detects_replacement(pair, tmp_path, monkeypatch):
from openultrasast.search import _sandbox
original = _sandbox.run
dependency = tmp_path / "dependency"
dependency.mkdir()
(dependency / "module.py").write_text("original")
sides, demo = pair
sides = [Side(s.checkout, s.command, dependencies=(dependency,)) for s in sides]
request(demo, "path")
def changed(job, **kwargs):
result = original(job, **kwargs)
if job.command[-1] == "/demo/build.sh":
(kwargs["mounts"]["/deps/0"] / "module.py").write_text("replacement")
return result
monkeypatch.setattr(_sandbox, "run", changed)
result = verify(*sides, demo, "path")
assert result.outcome == "inconclusive"
assert result.sides[0].reason == "build changed immutable inputs"
def test_trusted_build_products_are_used_readonly(pair):
sides, demo = pair
sides = [
Side(s.checkout, ("/usr/bin/python3", "-I", "/build/app.py"), build_command=("/bin/cp", "/workspace/app.py", "/scratch/app.py"))
for s in sides
]
request(demo, "path")
assert verify(*sides, demo, "path").outcome == "demonstrated"
(demo / "build.sh").write_text("echo patched > /build/app.py")
assert verify(*sides, demo, "path").outcome == "inconclusive"
def test_private_namespace_handoff_without_sockets(tmp_path):
import subprocess
from openultrasast.sandbox import SandboxJob
from openultrasast.search import _sandbox
# Proves the actual nsenter -> bwrap handoff without requiring socket().
process = subprocess.Popen(
[
"/usr/bin/unshare",
"--user",
"--map-root-user",
"--net",
"/usr/bin/python3",
"-I",
"-c",
"import os,time; print(os.getpid(), flush=True); time.sleep(10)",
],
stdout=subprocess.PIPE,
text=True,
)
try:
pid = int(process.stdout.readline())
scratch = tmp_path / "scratch"
scratch.mkdir()
job = SandboxJob("", ("/usr/bin/python3", "-I", "-c", 'print("namespace-ready")'), tmp_path, {}, 3, 256, 128)
result = _sandbox.run(job, scratch=scratch, namespace_pid=pid)
assert result.exit_code == 0, result.stderr
assert result.stdout.strip() == "namespace-ready"
finally:
process.terminate()
process.wait(timeout=3)
process.stdout.close()
def test_attacker_sees_only_external_bridge_and_scratch(pair):
sides, demo = pair
(demo / "attack.sh").write_text(
"test ! -e /workspace/app.py || exit 3\n"
"test ! -e /fixture/canary || exit 3\n"
"test ! -e /build/app.py || exit 3\n"
"test ! -e /deps/0 || exit 3\n"
'test -z "$CALLBACK_URL$PROOF_MARKER$DATABASE$SERVED_ROOT" || exit 3\n'
'"$TARGET" path ../canary\n'
)
assert verify(*sides, demo, "path").outcome == "demonstrated"
def test_listener_accepts_nonce_path_for_post_without_sockets(monkeypatch, capsys):
import fcntl
import http.server
import io
from openultrasast.search import oracles
class Socket:
def __enter__(self):
return self
def __exit__(self, *args):
pass
class Connection:
def __init__(self, path):
self.input = io.BytesIO(b"POST " + path + b" HTTP/1.0\r\nContent-Length: 0\r\n\r\n")
def makefile(self, *args):
return self.input
def sendall(self, data):
pass
class Server:
server_port = 12345
def __init__(self, address, handler):
self.handler = handler
def __enter__(self):
return self
def __exit__(self, *args):
pass
def serve_forever(self):
for path in (b"/wrong", b"/secret?query=1"):
self.handler(Connection(path), ("127.0.0.1", 1), self)
monkeypatch.setattr(socket, "socket", lambda *args: Socket())
monkeypatch.setattr(fcntl, "ioctl", lambda *args: None)
monkeypatch.setattr(http.server, "HTTPServer", Server)
oracles._listen("secret")
assert capsys.readouterr().out.splitlines()[1:] == ["observed"]