mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
search-with-proof: declarative demos, brain/executor split, kube-ax constraints
Demos are data the verifier executes (fixed build recipes, runtime+path start, HTTP/CLI steps with captures; the canary is never referable). The agent loop with the key never runs repository code: executor tasks without a key exchange commands and results through presigned objects. AX capacity errors are back-pressure with at most 2 tasks in flight; caches and builds under /workspace with a 2 GiB guard; verifiers take one presigned archive per side. Launcher fix: private /dev in bwrap. 190 tests on the host; probe 5/5 demonstrated. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
721c4b0997
commit
85f4f0b3d4
@@ -175,3 +175,154 @@ def test_search_environment(step):
|
||||
it.extra_env = {key: value}
|
||||
with pytest.raises(ValueError):
|
||||
manifest("ousast-engine-search-1", workload(), it, urls)
|
||||
|
||||
|
||||
def test_search_executor_task_links_and_cleanup():
|
||||
from types import SimpleNamespace
|
||||
|
||||
from benchmarks.ax.batch import SearchExecutorTask
|
||||
|
||||
class Store:
|
||||
def __init__(self):
|
||||
self.deleted = []
|
||||
self.uploads = []
|
||||
|
||||
def _put(self, key, data):
|
||||
self.uploads.append((key, data))
|
||||
|
||||
def presign_get(self, key, expires):
|
||||
return "https://files.example/" + key + "?get"
|
||||
|
||||
def presign_put(self, key, expires):
|
||||
return "https://files.example/" + key + "?put"
|
||||
|
||||
def _delete(self, key):
|
||||
self.deleted.append(key)
|
||||
|
||||
calls = []
|
||||
lane = SimpleNamespace(
|
||||
store=Store(),
|
||||
host="files.example",
|
||||
args=SimpleNamespace(atespace="default"),
|
||||
workload=workload(),
|
||||
ax=lambda *a, **kw: calls.append((a, kw)),
|
||||
clock=__import__("time").monotonic,
|
||||
pause=lambda _: None,
|
||||
)
|
||||
with SearchExecutorTask(lane, b"archive", deadline=60) as session:
|
||||
assert session.name.startswith("ousast-engine-search-exec-")
|
||||
assert session.client.command_url.endswith("?put")
|
||||
doc = calls[0][1]["manifest"]
|
||||
assert doc["spec"]["command"][2] == "openultrasast.search.executor"
|
||||
assert {r["name"] for r in doc["spec"]["env"]} == {"COMMAND_GET_URL", "EXECUTOR_RESULT_PUT_URL", "REPO_URL", "PREPARED_PUT_URL"}
|
||||
assert "API_KEY" not in json.dumps(doc)
|
||||
assert calls[-1][0][:2] == ("delete", "task")
|
||||
assert len(lane.store.deleted) == 4
|
||||
|
||||
|
||||
def test_capacity_backpressure_retries_same_manifest_until_deadline(tmp_path, monkeypatch):
|
||||
import subprocess
|
||||
from types import SimpleNamespace
|
||||
|
||||
from benchmarks.ax.batch import AXLane
|
||||
|
||||
now, applies, deletes = [0.0], [], []
|
||||
|
||||
class Store:
|
||||
def _put(self, *args):
|
||||
pass
|
||||
|
||||
def _get(self, *args):
|
||||
return (b"{}", None)
|
||||
|
||||
def _delete(self, *args):
|
||||
pass
|
||||
|
||||
def presign_get(self, key, expiry):
|
||||
return "https://files.example/" + key
|
||||
|
||||
presign_put = presign_get
|
||||
|
||||
busy = [True]
|
||||
|
||||
def run(argv, **kwargs):
|
||||
if argv[1] == "apply":
|
||||
applies.append(kwargs["input"])
|
||||
return subprocess.CompletedProcess(argv, 1 if busy[0] else 0, "", "no free workers available secret-url")
|
||||
if argv[1] == "delete":
|
||||
deletes.append(argv)
|
||||
return subprocess.CompletedProcess(argv, 0, "", "")
|
||||
|
||||
monkeypatch.setenv("S3_ENDPOINT", "https://files.example")
|
||||
monkeypatch.setattr(subprocess, "run", run)
|
||||
|
||||
def pause(seconds):
|
||||
assert 0 < seconds <= 30
|
||||
now[0] += seconds
|
||||
if now[0] > 30:
|
||||
busy[0] = False
|
||||
|
||||
def validate(data, output):
|
||||
(output / "result.json").write_text('{"status":"ok"}')
|
||||
|
||||
lane = AXLane(
|
||||
SimpleNamespace(ax_bin="ax", kubeconfig=None, atespace="default"),
|
||||
Workload(IMAGE, workload().url_env, 100, validate),
|
||||
Store(),
|
||||
pause=pause,
|
||||
clock=lambda: now[0],
|
||||
)
|
||||
assert lane(item(), tmp_path, 0)["status"] == "ok"
|
||||
assert len(applies) >= 3 and len(set(applies)) == 1
|
||||
assert len(deletes) == 1
|
||||
busy[0] = True
|
||||
lane.pause = lambda seconds: now.__setitem__(0, now[0] + seconds)
|
||||
deletes.clear()
|
||||
result = lane(item(), tmp_path, 1)
|
||||
assert result["status"] == "capacity_timeout"
|
||||
assert not deletes
|
||||
calls = []
|
||||
|
||||
def no_vm(*args):
|
||||
pytest.fail("capacity must not fall back")
|
||||
|
||||
def capacity(*args):
|
||||
calls.append(1)
|
||||
return result
|
||||
|
||||
assert dispatch([item()], workload(), tmp_path / "dispatch", {"ax": capacity, "docker": no_vm}, lanes=("ax", "docker"), overflow=False)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_shared_capacity_across_workloads(tmp_path, monkeypatch):
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
|
||||
from benchmarks.ax.batch import AXLane
|
||||
|
||||
monkeypatch.setenv("S3_ENDPOINT", "https://files.example")
|
||||
active, peak = [0], [0]
|
||||
lock = threading.Lock()
|
||||
|
||||
def execute(item, output, attempt, attempts):
|
||||
with lock:
|
||||
active[0] += 1
|
||||
peak[0] = max(peak[0], active[0])
|
||||
time.sleep(0.02)
|
||||
with lock:
|
||||
active[0] -= 1
|
||||
(output / "result.json").write_text('{"status":"ok"}')
|
||||
return subprocess.CompletedProcess([], 0), None
|
||||
|
||||
lanes = [AXLane(SimpleNamespace(), workload(), object()) for _ in range(6)]
|
||||
paths = [tmp_path / str(i) for i in range(6)]
|
||||
for lane, path in zip(lanes, paths, strict=True):
|
||||
path.mkdir()
|
||||
lane._execute_attempt = execute
|
||||
with ThreadPoolExecutor(max_workers=6) as pool:
|
||||
results = list(pool.map(lambda pair: pair[0](item(), pair[1], 0), zip(lanes, paths, strict=True)))
|
||||
assert all(r["status"] == "ok" for r in results)
|
||||
assert peak[0] == 2
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.search.demo import load_demo, validate_demo
|
||||
|
||||
|
||||
def document():
|
||||
return {
|
||||
"build": {"recipe": "none", "arguments": []},
|
||||
"start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
|
||||
"steps": [{"type": "cli", "arguments": ["hello"]}],
|
||||
}
|
||||
|
||||
|
||||
def test_valid_and_roundtrip(tmp_path):
|
||||
value = document()
|
||||
(tmp_path / "demo.json").write_text(json.dumps(value))
|
||||
assert load_demo(tmp_path) == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["../app.py", "/fixture/canary", "/etc/passwd", "a/../../app.py", "-c", "a\\b"])
|
||||
def test_checkout_paths_only(path):
|
||||
value = document()
|
||||
value["start"]["path"] = path
|
||||
with pytest.raises(ValueError):
|
||||
validate_demo(value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("argument", ["$PROOF_MARKER", "${CANARY}", "/fixture/canary", "/proc/1/environ", "${later}", "x\x00y"])
|
||||
def test_private_references_rejected(argument):
|
||||
value = document()
|
||||
value["steps"][0]["arguments"] = [argument]
|
||||
with pytest.raises(ValueError):
|
||||
validate_demo(value)
|
||||
|
||||
|
||||
def test_sequential_capture():
|
||||
value = document()
|
||||
value["steps"][0]["capture"] = {"name": "token", "source": "json", "path": ["token"]}
|
||||
value["steps"].append({"type": "cli", "arguments": ["${token}"]})
|
||||
assert validate_demo(value) == value
|
||||
value["steps"][0]["arguments"] = ["${token}"]
|
||||
with pytest.raises(ValueError):
|
||||
validate_demo(value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutate",
|
||||
[
|
||||
lambda d: d.update(env={}),
|
||||
lambda d: d["build"].update(recipe="shell"),
|
||||
lambda d: d["build"].update(arguments=["--index-url=evil"]),
|
||||
lambda d: d["start"].update(runtime="sh"),
|
||||
lambda d: d.update(steps=d["steps"] * 33),
|
||||
],
|
||||
)
|
||||
def test_unknown_keys_recipes_and_limits(mutate):
|
||||
value = document()
|
||||
mutate(value)
|
||||
with pytest.raises(ValueError):
|
||||
validate_demo(value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("recipe", ["npm", "composer", "gradle"])
|
||||
def test_directory_build_recipe_accepts_checkout_root(recipe):
|
||||
value = document()
|
||||
value["build"] = {"recipe": recipe, "arguments": ["."]}
|
||||
assert validate_demo(value) == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("recipe", ["pip", "maven"])
|
||||
def test_manifest_recipe_requires_file_path(recipe):
|
||||
value = document()
|
||||
value["build"] = {"recipe": recipe, "arguments": ["."]}
|
||||
with pytest.raises(ValueError):
|
||||
validate_demo(value)
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Offline executor boundary and mailbox protocol checks."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.search.executor import MAX_RESULT_BYTES, Command, InProcessExecutor, ObjectStoreExecutor, clean_environment, serve
|
||||
|
||||
|
||||
def test_order_retry_stop(tmp_path):
|
||||
(tmp_path / "input").write_text("hello")
|
||||
executor = InProcessExecutor(tmp_path)
|
||||
command = Command(1, "read_file", {"path": "input"})
|
||||
first = executor.execute(command)
|
||||
assert first["text"] == "hello" and first["input_bytes"] == 5
|
||||
(tmp_path / "input").write_text("changed")
|
||||
assert executor.execute(command) == first
|
||||
with pytest.raises(ValueError, match="sequence"):
|
||||
executor.execute(Command(3, "list_files", {}))
|
||||
with pytest.raises(ValueError, match="retry"):
|
||||
executor.execute(Command(1, "list_files", {}))
|
||||
assert executor.execute(Command(2, "stop", {}))["stopped"]
|
||||
with pytest.raises(ValueError, match="stopped"):
|
||||
executor.execute(Command(3, "list_files", {}))
|
||||
|
||||
|
||||
def test_paths_and_oversize(tmp_path):
|
||||
executor = InProcessExecutor(tmp_path)
|
||||
with pytest.raises(ValueError):
|
||||
executor.submit("read_file", {"path": "../secret"})
|
||||
(tmp_path / "huge").write_text("x" * 100000)
|
||||
result = executor.submit("read_file", {"path": "huge"}, limits={"result_bytes": 1024})
|
||||
assert result["truncated"]
|
||||
assert len(json.dumps(result).encode()) <= 1024
|
||||
with pytest.raises(ValueError):
|
||||
Command(1, "run", {}, timeout_seconds=0).validate()
|
||||
|
||||
|
||||
def test_environment_allowlist():
|
||||
assert clean_environment(
|
||||
{"OPENROUTER_API_KEY": "secret", "GEMINI_API_KEY": "secret", "PATH": "/evil", "AWS_SECRET_ACCESS_KEY": "secret"}
|
||||
) == {"PATH": "/usr/bin:/bin", "LANG": "C.UTF-8"}
|
||||
|
||||
|
||||
class Mailbox:
|
||||
def __init__(self, executor):
|
||||
self.executor, self.result, self.writes = executor, None, 0
|
||||
|
||||
def put(self, url, data, timeout):
|
||||
self.writes += 1
|
||||
result = self.executor.execute(Command(**json.loads(data)))
|
||||
self.result = json.dumps(result).encode()
|
||||
if self.writes == 1:
|
||||
raise TimeoutError("lost acknowledgement")
|
||||
|
||||
def get(self, url, timeout):
|
||||
return self.result
|
||||
|
||||
|
||||
def test_mailbox_retry_is_idempotent(tmp_path):
|
||||
mailbox = Mailbox(InProcessExecutor(tmp_path))
|
||||
client = ObjectStoreExecutor("https://store.example/command", "https://store.example/result", transport=mailbox, poll_seconds=0.001)
|
||||
assert client.submit("list_files", {})["files"] == []
|
||||
assert client.submit("stop", {})["stopped"]
|
||||
assert mailbox.writes >= 2
|
||||
|
||||
|
||||
def test_mailbox_timeout():
|
||||
class Empty:
|
||||
def put(self, *args):
|
||||
pass
|
||||
|
||||
def get(self, *args):
|
||||
return None
|
||||
|
||||
client = ObjectStoreExecutor("https://store.example/command", "https://store.example/result", transport=Empty(), poll_seconds=0.001)
|
||||
with pytest.raises(TimeoutError):
|
||||
client.submit("list_files", {}, timeout_seconds=0.01)
|
||||
with pytest.raises(RuntimeError, match="uncertain"):
|
||||
client.submit("list_files", {})
|
||||
|
||||
|
||||
def test_serve_retries_result_without_reexecuting(tmp_path):
|
||||
executor = InProcessExecutor(tmp_path)
|
||||
|
||||
class Transport:
|
||||
writes = 0
|
||||
|
||||
def get(self, url, timeout):
|
||||
return json.dumps({"seq": 1, "name": "stop", "args": {}, "timeout_seconds": 0.01}).encode()
|
||||
|
||||
def put(self, url, data, timeout):
|
||||
self.writes += 1
|
||||
if self.writes == 1:
|
||||
raise TimeoutError()
|
||||
assert json.loads(data)["stopped"]
|
||||
|
||||
transport = Transport()
|
||||
serve(executor, "command", "result", transport=transport, deadline_seconds=0.2, poll_seconds=0.001)
|
||||
assert transport.writes == 2
|
||||
|
||||
|
||||
def test_local_run_fails_closed_without_namespaces(tmp_path, monkeypatch):
|
||||
from openultrasast.search import _sandbox
|
||||
|
||||
def unavailable(**kwargs):
|
||||
raise _sandbox.IsolationUnavailable("no namespaces")
|
||||
|
||||
monkeypatch.setattr(_sandbox, "isolation_check", unavailable)
|
||||
with pytest.raises(_sandbox.IsolationUnavailable):
|
||||
InProcessExecutor(tmp_path).submit("run", {"command": ["/bin/true"]})
|
||||
|
||||
|
||||
def test_task_run_child_environment_is_key_free(tmp_path, monkeypatch):
|
||||
from openultrasast.search import _sandbox
|
||||
from openultrasast.search import executor as module
|
||||
|
||||
def unavailable(**kwargs):
|
||||
raise _sandbox.IsolationUnavailable("no namespaces")
|
||||
|
||||
monkeypatch.setattr(_sandbox, "isolation_check", unavailable)
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "secret")
|
||||
seen = {}
|
||||
|
||||
class Process:
|
||||
pid = 123456789
|
||||
returncode = 0
|
||||
|
||||
def __init__(self, argv, **kwargs):
|
||||
seen.update(kwargs)
|
||||
|
||||
def wait(self, **kwargs):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(module.subprocess, "Popen", Process)
|
||||
monkeypatch.setattr(module.os, "killpg", lambda *args: None)
|
||||
result = InProcessExecutor(tmp_path, task_boundary=True).submit("run", {"command": ["/bin/true"]})
|
||||
assert result["isolation_mode"] == "task-boundary"
|
||||
assert seen["env"] == clean_environment()
|
||||
assert seen["cwd"] == tmp_path
|
||||
|
||||
|
||||
def test_mailbox_oversize_and_future_sequence():
|
||||
class Response:
|
||||
value = b"x" * (MAX_RESULT_BYTES + 1)
|
||||
|
||||
def put(self, *args):
|
||||
pass
|
||||
|
||||
def get(self, *args):
|
||||
return self.value
|
||||
|
||||
response = Response()
|
||||
client = ObjectStoreExecutor("https://store.example/command", "https://store.example/result", transport=response)
|
||||
with pytest.raises(ValueError, match="large"):
|
||||
client.submit("list_files", {})
|
||||
response.value = b'{"seq": 9}'
|
||||
client = ObjectStoreExecutor("https://store.example/command", "https://store.example/result", transport=response)
|
||||
with pytest.raises(ValueError, match="future"):
|
||||
client.submit("list_files", {})
|
||||
|
||||
|
||||
def test_archive_rejects_links_and_parent_paths(tmp_path):
|
||||
import io
|
||||
import tarfile
|
||||
|
||||
from openultrasast.search.executor import unpack_checkout
|
||||
|
||||
for index, name in enumerate(("../escape", "/outside", ".git/config", "link")):
|
||||
content = io.BytesIO()
|
||||
with tarfile.open(fileobj=content, mode="w") as archive:
|
||||
member = tarfile.TarInfo(name)
|
||||
if name == "link":
|
||||
member.type = tarfile.SYMTYPE
|
||||
member.linkname = "/outside"
|
||||
archive.addfile(member)
|
||||
with pytest.raises(ValueError, match="unsafe"):
|
||||
unpack_checkout(content.getvalue(), tmp_path / str(index))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", ["OPENROUTER_API_KEY", "GEMINI_API_KEY", "OPENAI_KEY", "DEEPSEEK_TOKEN", "GEMINI_TOKEN"])
|
||||
def test_entrypoint_rejects_initial_credentials(tmp_path, monkeypatch, key):
|
||||
import sys
|
||||
|
||||
from openultrasast.search.executor import main
|
||||
|
||||
monkeypatch.setattr(sys, "argv", ["executor", "--repo", str(tmp_path)])
|
||||
monkeypatch.setenv(key, "sentinel-not-to-print")
|
||||
with pytest.raises(SystemExit, match="refuses inherited credentials"):
|
||||
main()
|
||||
|
||||
|
||||
def test_stale_result_is_polled_until_matching_sequence(tmp_path):
|
||||
class Stale(Mailbox):
|
||||
def get(self, url, timeout):
|
||||
if self.writes == 1:
|
||||
return b'{"seq": 0, "files": ["stale"]}'
|
||||
return self.result
|
||||
|
||||
def put(self, url, data, timeout):
|
||||
self.writes += 1
|
||||
self.result = json.dumps(self.executor.execute(Command(**json.loads(data)))).encode()
|
||||
|
||||
transport = Stale(InProcessExecutor(tmp_path))
|
||||
client = ObjectStoreExecutor("https://store.example/command", "https://store.example/result", transport=transport, poll_seconds=0.001)
|
||||
assert client.submit("list_files", {})["files"] == []
|
||||
assert transport.writes == 2
|
||||
|
||||
|
||||
def test_task_boundary_run_honors_shorter_argument_timeout(tmp_path, monkeypatch):
|
||||
import time
|
||||
|
||||
from openultrasast.search import _sandbox
|
||||
|
||||
def unavailable(**kwargs):
|
||||
raise _sandbox.IsolationUnavailable("no namespaces")
|
||||
|
||||
monkeypatch.setattr(_sandbox, "isolation_check", unavailable)
|
||||
started = time.monotonic()
|
||||
result = InProcessExecutor(tmp_path, task_boundary=True).submit(
|
||||
"run", {"command": ["/bin/sleep", "2"], "timeout_seconds": 1}, timeout_seconds=4
|
||||
)
|
||||
assert result["timed_out"] is True
|
||||
assert result["exit_code"] != 0
|
||||
assert time.monotonic() - started < 1.8
|
||||
|
||||
|
||||
def test_sandbox_receives_exact_remaining_wall_limit(tmp_path, monkeypatch):
|
||||
from openultrasast.sandbox import SandboxResult
|
||||
from openultrasast.search import _sandbox
|
||||
|
||||
seen = {}
|
||||
monkeypatch.setattr(_sandbox, "isolation_check", lambda **kwargs: seen.update(probe=kwargs["timeout_seconds"]))
|
||||
|
||||
def run(job, **kwargs):
|
||||
seen.update(wall=kwargs.get("wall_timeout_seconds"))
|
||||
return SandboxResult(0, "", "", False)
|
||||
|
||||
monkeypatch.setattr(_sandbox, "run", run)
|
||||
InProcessExecutor(tmp_path).submit("run", {"command": ["/bin/true"]}, timeout_seconds=0.2)
|
||||
assert 0 < seen["probe"] <= 0.2
|
||||
assert 0 < seen["wall"] <= 0.2
|
||||
@@ -38,6 +38,9 @@ def test_root_argv(monkeypatch, tmp_path, pid):
|
||||
def wait(self, **kwargs):
|
||||
return 0
|
||||
|
||||
def communicate(self, **kwargs):
|
||||
return None, None
|
||||
|
||||
monkeypatch.setattr(_sandbox.subprocess, "Popen", Process)
|
||||
job = SandboxJob("", ("/bin/true",), tmp_path, {}, 5, 256, 128)
|
||||
_sandbox.run(job, scratch=tmp_path, namespace_pid=pid)
|
||||
@@ -91,7 +94,8 @@ def test_namespace_diagnostics_precede_failed_preflight(monkeypatch, tmp_path):
|
||||
assert record["status"] == "instrument_failure"
|
||||
assert set(record["namespaces"]) == {"user", "pid", "net", "ipc", "uts", "cgroup", "mount", "setpriv"}
|
||||
assert all(row["exit_code"] == 1 and row["stderr"] == "denied" for row in record["namespaces"].values())
|
||||
assert record["isolation_mode"] == _sandbox.isolation_mode()
|
||||
assert record["isolation_mode"] == "task-boundary"
|
||||
assert record["requires_fresh_side_tasks"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("uid", [0, 1000])
|
||||
@@ -158,3 +162,22 @@ def test_root_fixture_permissions_and_browser_records(monkeypatch, tmp_path):
|
||||
assert record.isolation_mode == "root-no-userns"
|
||||
assert record.chromium_no_sandbox
|
||||
assert all(s.chromium_no_sandbox and s.isolation_mode == "root-no-userns" for s in record.sides)
|
||||
|
||||
|
||||
def test_devnull_can_be_opened_by_nested_launcher(tmp_path):
|
||||
"""The HTTP driver's Popen opens /dev/null inside bwrap, before any socket."""
|
||||
_sandbox.isolation_check()
|
||||
source = tmp_path / "input"
|
||||
source.write_text("readable")
|
||||
scratch = tmp_path / "scratch"
|
||||
scratch.mkdir()
|
||||
code = (
|
||||
"from pathlib import Path; import subprocess; "
|
||||
"print(Path('/workspace/input').stat().st_size); "
|
||||
"p=subprocess.Popen(['/bin/true'], stdin=subprocess.DEVNULL, "
|
||||
"stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); "
|
||||
"raise SystemExit(p.wait())"
|
||||
)
|
||||
result = _sandbox.run(SandboxJob("", ("/usr/bin/python3", "-I", "-c", code), tmp_path, {}, 5, 256, 128), scratch=scratch)
|
||||
assert result.stdout.strip() == "8", result
|
||||
assert result.exit_code == 0 and not result.timed_out, result
|
||||
|
||||
@@ -28,7 +28,7 @@ def test_packaged_pairs_and_three_demos(tmp_path):
|
||||
assert len(demos) == 3
|
||||
assert all((s.checkout / "app.py").stat().st_size > 0 for s in sides)
|
||||
assert (sides[1].checkout / "fixed").exists()
|
||||
assert json.loads((demos["real"] / "request.json").read_text())[0] == family
|
||||
assert json.loads((demos["real"] / "demo.json").read_text())["steps"][0]["arguments"][0] == family
|
||||
|
||||
|
||||
def test_batch_workload_and_all_lanes(tmp_path, monkeypatch):
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Offline operator storage/egress contracts, using real local child processes."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.search import executor, task_storage, verify_task
|
||||
from openultrasast.search.verify import Side, verify_side_task
|
||||
|
||||
|
||||
def spec():
|
||||
return {
|
||||
"demo": {
|
||||
"build": {"recipe": "none", "arguments": []},
|
||||
"start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
|
||||
"steps": [{"type": "cli", "arguments": []}],
|
||||
},
|
||||
"family": "path",
|
||||
"timeout_seconds": 2,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workspace(tmp_path, monkeypatch):
|
||||
root = tmp_path / "workspace"
|
||||
root.mkdir()
|
||||
monkeypatch.setattr(task_storage, "WORKSPACE", root)
|
||||
# Keep global tempfile state and environment changes local to this test.
|
||||
monkeypatch.setattr(task_storage.tempfile, "tempdir", str(tmp_path))
|
||||
monkeypatch.setattr(os, "environ", dict(os.environ))
|
||||
task_storage.configure()
|
||||
return root
|
||||
|
||||
|
||||
def test_workspace_environment_and_cached_tempdir(workspace):
|
||||
env = executor.clean_environment()
|
||||
assert env["TMPDIR"] == str(workspace / "tmp")
|
||||
assert task_storage.tempfile.gettempdir() == env["TMPDIR"]
|
||||
for key, suffix in {
|
||||
"npm_config_cache": ".npm",
|
||||
"COMPOSER_HOME": ".composer",
|
||||
"PIP_CACHE_DIR": ".pip",
|
||||
"GRADLE_USER_HOME": ".gradle",
|
||||
}.items():
|
||||
assert env[key] == str(workspace / suffix)
|
||||
assert env["MAVEN_OPTS"] == "-Dmaven.repo.local=" + str(workspace / ".m2")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wait", [False, True])
|
||||
def test_executor_checks_caches_and_stops_at_scratch_limit(workspace, monkeypatch, wait):
|
||||
repo = workspace / "checkout"
|
||||
repo.mkdir()
|
||||
(repo / "input").write_text("readable-input")
|
||||
monkeypatch.setenv("OUSAST_SCRATCH_BYTES", "20000")
|
||||
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||
code = (
|
||||
"from pathlib import Path; import os,time; print(Path('input').stat().st_size); "
|
||||
"p=Path(os.environ['PIP_CACHE_DIR']); p.mkdir(); (p/'large').write_bytes(b'x'*30000); " + ("time.sleep(10)" if wait else "")
|
||||
)
|
||||
result = app.submit("run", {"command": [sys.executable, "-c", code]}, timeout_seconds=3)
|
||||
assert result["status"] == "could_not_build" and result["reason"] == "scratch limit"
|
||||
assert result["stopped"] and app.stopped
|
||||
|
||||
|
||||
def test_executor_exports_built_input_and_verifier_needs_no_build(workspace, monkeypatch):
|
||||
repo = workspace / "checkout"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("print('safe')")
|
||||
blobs = []
|
||||
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||
app.prepared_put_url = "https://files.example/output"
|
||||
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
|
||||
result = app.submit("prepare_verification", spec())
|
||||
assert result["status"] == "ok", result
|
||||
destination = workspace / "unpacked"
|
||||
verify_task.extract_checkout(blobs[0], destination)
|
||||
assert (destination / "checkout/app.py").read_text() == "print('safe')"
|
||||
assert json.loads((destination / "spec.json").read_text()) == spec()
|
||||
assert (destination / "products").is_dir()
|
||||
demo = workspace / "demo"
|
||||
demo.mkdir()
|
||||
(demo / "demo.json").write_text(json.dumps(spec()["demo"]))
|
||||
observation = verify_side_task(Side(destination / "checkout", products=destination / "products"), demo, "path")
|
||||
assert observation.outcome == "observed", observation
|
||||
|
||||
|
||||
def test_verifier_refuses_build_recipe():
|
||||
value = spec()
|
||||
value["demo"]["build"] = {"recipe": "npm", "arguments": ["project"]}
|
||||
with pytest.raises(ValueError, match="builds are forbidden"):
|
||||
verify_task.validate_spec(value)
|
||||
|
||||
|
||||
def test_first_download_retries_for_60_seconds(monkeypatch):
|
||||
import urllib.error
|
||||
|
||||
now, timeouts = [0.0], []
|
||||
|
||||
class Opener:
|
||||
def open(self, url, timeout):
|
||||
timeouts.append(timeout)
|
||||
raise urllib.error.HTTPError(url, 403, "pending policy", {}, None)
|
||||
|
||||
monkeypatch.setattr(verify_task.urllib.request, "build_opener", lambda *a: Opener())
|
||||
monkeypatch.setattr(verify_task.time, "monotonic", lambda: now[0])
|
||||
monkeypatch.setattr(verify_task.time, "sleep", lambda seconds: now.__setitem__(0, now[0] + seconds))
|
||||
with pytest.raises(OSError, match="egress not ready"):
|
||||
verify_task.download("https://files.because-security.com/input", 100)
|
||||
assert now[0] == 60 and len(timeouts) > 3
|
||||
|
||||
|
||||
def test_verifier_reports_scratch_limit(workspace, monkeypatch):
|
||||
repo = workspace / "checkout"
|
||||
repo.mkdir()
|
||||
# Writes happen after readiness, in a package cache outside the checkout.
|
||||
(repo / "app.py").write_text(
|
||||
"import sys,os\nfrom pathlib import Path\n"
|
||||
"if '--ready' not in sys.argv:\n"
|
||||
" p=Path(os.environ['npm_config_cache']); p.mkdir(exist_ok=True); (p/'large').write_bytes(b'x'*30000)\n"
|
||||
)
|
||||
demo = workspace / "demo"
|
||||
demo.mkdir()
|
||||
(demo / "demo.json").write_text(json.dumps(spec()["demo"]))
|
||||
monkeypatch.setenv("OUSAST_SCRATCH_BYTES", "25000")
|
||||
result = verify_side_task(Side(repo), demo, "path")
|
||||
assert result.outcome == "could_not_build" and result.reason == "scratch limit", result
|
||||
|
||||
|
||||
def test_preparation_builds_before_export_and_exports_products(workspace, monkeypatch):
|
||||
repo = workspace / "checkout"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("import dependency; print(dependency.VALUE)")
|
||||
(repo / "requirements.txt").write_text("dependency==1")
|
||||
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||
app.prepared_put_url = "https://files.example/output"
|
||||
blobs, commands = [], []
|
||||
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
|
||||
|
||||
def build(argv, timeout, limits):
|
||||
commands.append(argv)
|
||||
products = Path(argv[argv.index("--target") + 1])
|
||||
assert products.is_relative_to(workspace)
|
||||
products.mkdir()
|
||||
(products / "dependency.py").write_text("VALUE = 42")
|
||||
return {"exit_code": 0, "timed_out": False, "stderr": ""}
|
||||
|
||||
monkeypatch.setattr(app, "_task_run", build)
|
||||
request = spec()
|
||||
request["demo"]["build"] = {"recipe": "pip", "arguments": ["requirements.txt"]}
|
||||
result = app.submit("prepare_verification", request)
|
||||
assert result["status"] == "ok", result
|
||||
assert len(commands) == 1 and "--no-index" not in commands[0]
|
||||
unpacked = workspace / "unpacked"
|
||||
verify_task.extract_checkout(blobs[0], unpacked)
|
||||
assert (unpacked / "products/packages/dependency.py").read_text() == "VALUE = 42"
|
||||
assert json.loads((unpacked / "spec.json").read_text())["demo"]["build"] == {"recipe": "none", "arguments": []}
|
||||
|
||||
|
||||
def test_open_logs_count_toward_scratch_guard(workspace, monkeypatch):
|
||||
repo = workspace / "checkout"
|
||||
repo.mkdir()
|
||||
monkeypatch.setenv("OUSAST_SCRATCH_BYTES", "20000")
|
||||
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||
result = app.submit("run", {"command": [sys.executable, "-c", "print('x'*30000)"]})
|
||||
assert result["status"] == "could_not_build" and result["reason"] == "scratch limit"
|
||||
|
||||
|
||||
def test_unreadable_scratch_is_not_reported_as_empty(tmp_path):
|
||||
with pytest.raises(OSError, match="scratch root unavailable"):
|
||||
task_storage.check(tmp_path / "missing")
|
||||
+294
-231
@@ -1,106 +1,78 @@
|
||||
"""Owned observations and adversarial artefacts, with real isolated toy processes."""
|
||||
"""Schema-only verification, owned observations and adversarial public inputs."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import json
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.search.verify import Side, verify
|
||||
|
||||
FIXTURE = Path(__file__).parents[1] / "src/openultrasast/search/probe_apps/app.py.txt"
|
||||
PAYLOADS = {
|
||||
"path": "../canary",
|
||||
"sql": "' OR public=0 --",
|
||||
"command": '; touch "$PROOF_MARKER"',
|
||||
"ssrf": "$CALLBACK_URL",
|
||||
}
|
||||
from openultrasast.search import _sandbox
|
||||
from openultrasast.search.probe import materialise
|
||||
from openultrasast.search.verify import RunObservation, SideRecord, verify, verify_side_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pair(tmp_path):
|
||||
sides = []
|
||||
for name in ("vulnerable", "fixed"):
|
||||
root = tmp_path / name
|
||||
root.mkdir()
|
||||
shutil.copyfile(FIXTURE, root / "app.py")
|
||||
if name == "fixed":
|
||||
(root / "fixed").touch()
|
||||
sides.append(Side(root, ("/usr/bin/python3", "-I", "/workspace/app.py")))
|
||||
demo = tmp_path / "demo"
|
||||
demo.mkdir()
|
||||
(demo / "build.sh").write_text("test -s /workspace/app.py\n")
|
||||
return sides, demo
|
||||
sides, demos = materialise(tmp_path / "pair", "path")
|
||||
return sides, demos["real"]
|
||||
|
||||
|
||||
def request(demo, family):
|
||||
import json
|
||||
|
||||
(demo / "request.json").write_text(json.dumps([family, PAYLOADS[family]]))
|
||||
def edit(demo, change):
|
||||
path = demo / "demo.json"
|
||||
value = json.loads(path.read_text())
|
||||
change(value)
|
||||
path.write_text(json.dumps(value))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("family", ["path", "sql", "command", "ssrf"])
|
||||
def test_real_differential(pair, family):
|
||||
def test_real_differential(tmp_path, family):
|
||||
if family == "ssrf":
|
||||
try:
|
||||
with socket.socket():
|
||||
pass
|
||||
except PermissionError:
|
||||
pytest.skip("host policy prohibits even localhost sockets")
|
||||
sides, demo = pair
|
||||
request(demo, family)
|
||||
result = verify(*sides, demo, family)
|
||||
pytest.skip("host policy prohibits localhost sockets")
|
||||
sides, demos = materialise(tmp_path / family, family)
|
||||
assert (sides[0].checkout / "app.py").stat().st_size > 0
|
||||
result = verify(*sides, demos["real"], family)
|
||||
assert result.outcome == "demonstrated", result
|
||||
assert [r.observed for r in result.sides[0].runs] == [True] * 3
|
||||
assert [r.observed for r in result.sides[1].runs] == [False] * 3
|
||||
assert result.elapsed_seconds > 0
|
||||
assert result.isolation_mode == _sandbox.isolation_mode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"script",
|
||||
[
|
||||
"cat /fixture/canary; echo success; exit 42",
|
||||
"if test ! -f /workspace/fixed; then echo demonstrated; fi",
|
||||
"echo hacked > /workspace/sitecustomize.py",
|
||||
"echo hacked > /usr/lib/python3.12/sitecustomize.py",
|
||||
"echo hacked > /deps/0/module.py",
|
||||
"cat /proc/1/environ; echo demonstrated",
|
||||
'/usr/bin/python3 -I -c "import socket; socket.create_connection(("127.0.0.1",12345))"',
|
||||
],
|
||||
)
|
||||
def test_gaming_cannot_demonstrate(pair, tmp_path, script):
|
||||
@pytest.mark.parametrize("argument", ["/fixture/canary", "/proc/1/environ", "${CANARY}", "$CALLBACK_URL"])
|
||||
def test_gaming_private_references_rejected_before_process(pair, argument, monkeypatch):
|
||||
sides, demo = pair
|
||||
deps = tmp_path / "deps"
|
||||
deps.mkdir()
|
||||
(deps / "module.py").write_text("original")
|
||||
sides = [Side(s.checkout, s.command, dependencies=(deps,)) for s in sides]
|
||||
(demo / "attack.sh").write_text(script)
|
||||
edit(demo, lambda value: value["steps"][0].update(arguments=[argument]))
|
||||
monkeypatch.setattr(_sandbox, "run", lambda *a, **kw: pytest.fail("invalid demo spawned a process"))
|
||||
result = verify(*sides, demo, "path")
|
||||
assert result.outcome != "demonstrated"
|
||||
assert (deps / "module.py").read_text() == "original"
|
||||
assert not (sides[0].checkout / "sitecustomize.py").exists()
|
||||
assert result.outcome == "inconclusive"
|
||||
assert "invalid declarative demo" in result.reason
|
||||
|
||||
|
||||
def test_attack_exit_and_stdout_ignored(pair):
|
||||
def test_legacy_scripts_rejected_before_process(pair, monkeypatch):
|
||||
sides, demo = pair
|
||||
(demo / "attack.sh").write_text('"$TARGET" path ../canary\necho fake\nexit 99\n')
|
||||
assert verify(*sides, demo, "path").outcome == "demonstrated"
|
||||
(demo / "attack.sh").write_text("echo demonstrated")
|
||||
monkeypatch.setattr(_sandbox, "run", lambda *a, **kw: pytest.fail("script executed"))
|
||||
assert verify(*sides, demo, "path").outcome == "inconclusive"
|
||||
|
||||
|
||||
def test_safe_build_failure_is_inconclusive(pair):
|
||||
def test_safe_build_failure_inconclusive(pair):
|
||||
sides, demo = pair
|
||||
request(demo, "path")
|
||||
(demo / "build.sh").write_text("test ! -f /workspace/fixed")
|
||||
(sides[1].checkout / "app.py").unlink()
|
||||
result = verify(*sides, demo, "path")
|
||||
assert result.outcome == "inconclusive"
|
||||
assert result.sides[1].outcome == "could_not_build"
|
||||
|
||||
|
||||
def test_affected_build_failure(pair):
|
||||
sides, demo = pair
|
||||
(sides[0].checkout / "app.py").unlink()
|
||||
assert verify(*sides, demo, "path").outcome == "could_not_build"
|
||||
|
||||
|
||||
def test_readiness_failure(pair):
|
||||
sides, demo = pair
|
||||
request(demo, "path")
|
||||
(sides[1].checkout / "broken").touch()
|
||||
result = verify(*sides, demo, "path")
|
||||
assert result.outcome == "inconclusive"
|
||||
@@ -113,140 +85,29 @@ def test_no_oracle(pair, family):
|
||||
assert verify(*sides, demo, family).outcome == "no_oracle"
|
||||
|
||||
|
||||
def test_flaky_observation_is_inconclusive(pair, monkeypatch):
|
||||
def test_flaky_observation_inconclusive(pair, monkeypatch):
|
||||
from openultrasast.search import oracles
|
||||
|
||||
nonces = iter(["a" * 48, "b" * 48, "c" * 48] * 2)
|
||||
monkeypatch.setattr(oracles.secrets, "token_hex", lambda _: next(nonces))
|
||||
sides, demo = pair
|
||||
(sides[0].checkout / "flaky").touch()
|
||||
request(demo, "path")
|
||||
result = verify(*sides, demo, "path")
|
||||
assert result.outcome == "inconclusive"
|
||||
assert [r.observed for r in result.sides[0].runs] == [True, False, False]
|
||||
|
||||
|
||||
def test_environment_injection_stays_in_artefact(pair, monkeypatch):
|
||||
def test_no_inherited_environment(pair, monkeypatch):
|
||||
for key in ("PYTHONPATH", "NODE_OPTIONS", "LD_PRELOAD", "GEMINI_API_KEY"):
|
||||
monkeypatch.setenv(key, "/bogus-key")
|
||||
sides, demo = pair
|
||||
monkeypatch.setenv("PYTHONPATH", "/bogus")
|
||||
monkeypatch.setenv("NODE_OPTIONS", "--require=/bogus")
|
||||
monkeypatch.setenv("LD_PRELOAD", "/bogus")
|
||||
(demo / "attack.sh").write_text(
|
||||
'test -z "$PYTHONPATH$NODE_OPTIONS$LD_PRELOAD" || exit 3\n'
|
||||
'echo "raise RuntimeError()" > /scratch/sitecustomize.py\n'
|
||||
"export PYTHONPATH=/scratch NODE_OPTIONS=--require=/scratch/sitecustomize.py LD_PRELOAD=/bogus\n"
|
||||
'"$TARGET" path ../canary\n'
|
||||
)
|
||||
assert verify(*sides, demo, "path").outcome == "demonstrated"
|
||||
|
||||
|
||||
def test_untracked_file_outside_scratch_refused(pair):
|
||||
sides, demo = pair
|
||||
(demo / "attack.sh").write_text('echo hacked > /untracked\n"$TARGET" path ../canary\n')
|
||||
assert verify(*sides, demo, "path").outcome == "inconclusive"
|
||||
|
||||
|
||||
def test_unknown_app_failure_cannot_prove_effect(pair):
|
||||
sides, demo = pair
|
||||
(sides[0].checkout / "app.py").write_text('raise RuntimeError("no app")')
|
||||
request(demo, "path")
|
||||
assert verify(*sides, demo, "path").outcome == "could_not_run"
|
||||
|
||||
|
||||
def test_affected_build_failure(pair):
|
||||
sides, demo = pair
|
||||
request(demo, "path")
|
||||
(demo / "build.sh").write_text("test -f /workspace/fixed")
|
||||
assert verify(*sides, demo, "path").outcome == "could_not_build"
|
||||
|
||||
|
||||
def test_missing_isolation_fails_closed(pair, monkeypatch):
|
||||
from openultrasast.search import _sandbox
|
||||
|
||||
def unavailable(*args, **kwargs):
|
||||
raise FileNotFoundError("bwrap unavailable")
|
||||
|
||||
monkeypatch.setattr(_sandbox, "run", unavailable)
|
||||
sides, demo = pair
|
||||
request(demo, "path")
|
||||
with pytest.raises(_sandbox.IsolationUnavailable, match="bwrap unavailable"):
|
||||
verify(*sides, demo, "path")
|
||||
|
||||
|
||||
def test_bwrap_failure_raises_before_product_outcomes(pair, monkeypatch):
|
||||
from openultrasast.sandbox import SandboxResult
|
||||
from openultrasast.search import _sandbox
|
||||
|
||||
calls = []
|
||||
stderr = "bwrap: Creating new namespace failed: Resource temporarily unavailable"
|
||||
|
||||
def failed(job, **kwargs):
|
||||
calls.append(job)
|
||||
return SandboxResult(1, "", stderr, False)
|
||||
|
||||
monkeypatch.setattr(_sandbox, "run", failed)
|
||||
sides, demo = pair
|
||||
with pytest.raises(_sandbox.IsolationUnavailable, match=stderr):
|
||||
verify(*sides, demo, "path")
|
||||
assert len(calls) == 1
|
||||
assert calls[0].command == ("/bin/cat", "/workspace/probe")
|
||||
|
||||
|
||||
def test_verification_with_many_host_processes(pair, monkeypatch):
|
||||
from openultrasast.search import _sandbox
|
||||
|
||||
checks = []
|
||||
original = _sandbox.isolation_check
|
||||
|
||||
def checked(**kwargs):
|
||||
checks.append(True)
|
||||
return original(**kwargs)
|
||||
|
||||
monkeypatch.setattr(_sandbox, "isolation_check", checked)
|
||||
sides, demo = pair
|
||||
request(demo, "path")
|
||||
children = []
|
||||
try:
|
||||
for _ in range(300):
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
try:
|
||||
while True:
|
||||
signal.pause()
|
||||
finally:
|
||||
os._exit(0)
|
||||
children.append(pid)
|
||||
assert verify(*sides, demo, "path").outcome == "demonstrated"
|
||||
assert checks == [True]
|
||||
finally:
|
||||
for pid in children:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
for pid in children:
|
||||
os.waitpid(pid, 0)
|
||||
|
||||
|
||||
def test_symlink_input_refused(pair):
|
||||
sides, demo = pair
|
||||
(demo / "escape").symlink_to("/etc/passwd")
|
||||
request(demo, "path")
|
||||
assert verify(*sides, demo, "path").outcome == "inconclusive"
|
||||
|
||||
|
||||
def test_xss_executor_contract(tmp_path):
|
||||
from openultrasast.search.oracles import Canary, oracle_for
|
||||
|
||||
calls = []
|
||||
|
||||
def browser(document, nonce):
|
||||
calls.append((document, nonce))
|
||||
return document == "<script>execute()</script>"
|
||||
|
||||
oracle = oracle_for("xss", browser)
|
||||
canary = Canary.fresh(tmp_path / "fixture")
|
||||
assert oracle.prepare(canary) == {}
|
||||
oracle.capture("<script>execute()</script>")
|
||||
assert oracle.observe()[0]
|
||||
assert calls == [("<script>execute()</script>", canary.nonce)]
|
||||
(sides[0].checkout / "escape").symlink_to("/etc/passwd")
|
||||
assert verify(*sides, demo, "path").outcome != "demonstrated"
|
||||
|
||||
|
||||
def test_six_fresh_canaries(pair, monkeypatch):
|
||||
@@ -261,59 +122,109 @@ def test_six_fresh_canaries(pair, monkeypatch):
|
||||
|
||||
monkeypatch.setattr(oracles.PathOracle, "prepare", record)
|
||||
sides, demo = pair
|
||||
request(demo, "path")
|
||||
assert verify(*sides, demo, "path").outcome == "demonstrated"
|
||||
assert len(set(nonces)) == 6
|
||||
|
||||
|
||||
def test_readiness_effect_cannot_prove_ssrf(pair, monkeypatch):
|
||||
from openultrasast.search import verify as module
|
||||
from openultrasast.search.oracles import OutputOracle
|
||||
def test_missing_isolation_requires_fresh_task_dispatcher(pair, monkeypatch):
|
||||
def failed(**kwargs):
|
||||
raise _sandbox.IsolationUnavailable("namespace denied")
|
||||
|
||||
class ReadinessOracle(OutputOracle):
|
||||
def observe(self):
|
||||
return True, "request during readiness"
|
||||
|
||||
monkeypatch.setattr(module, "oracle_for", lambda *args: ReadinessOracle())
|
||||
monkeypatch.setattr(_sandbox, "isolation_check", failed)
|
||||
sides, demo = pair
|
||||
result = module.verify(*sides, demo, "ssrf")
|
||||
assert result.outcome == "inconclusive"
|
||||
assert not result.sides[0].runs
|
||||
with pytest.raises(_sandbox.IsolationUnavailable, match="fresh-task dispatcher"):
|
||||
verify(*sides, demo, "path")
|
||||
calls = []
|
||||
|
||||
def dispatch(**kwargs):
|
||||
calls.append(kwargs)
|
||||
observed = kwargs["side"] == sides[0]
|
||||
return SideRecord("observed", (RunObservation(observed, "", 0),) * 3, 0, "", isolation_mode="task-boundary")
|
||||
|
||||
result = verify(*sides, demo, "path", task_dispatcher=dispatch)
|
||||
assert result.outcome == "demonstrated"
|
||||
assert result.isolation_mode == "task-boundary"
|
||||
assert len(calls) == 2 and all(call["fresh_task"] for call in calls)
|
||||
|
||||
|
||||
def test_dependency_hash_detects_replacement(pair, tmp_path, monkeypatch):
|
||||
from openultrasast.search import _sandbox
|
||||
|
||||
original = _sandbox.run
|
||||
dependency = tmp_path / "dependency"
|
||||
dependency.mkdir()
|
||||
(dependency / "module.py").write_text("original")
|
||||
def test_task_side_runs_one_repetition_without_model_environment(pair, monkeypatch):
|
||||
sides, demo = pair
|
||||
sides = [Side(s.checkout, s.command, dependencies=(dependency,)) for s in sides]
|
||||
request(demo, "path")
|
||||
monkeypatch.setenv("GEMINI_API_KEY", "must-not-leak")
|
||||
for side in sides:
|
||||
source = side.checkout / "app.py"
|
||||
source.write_text('import os\nassert "GEMINI_API_KEY" not in os.environ\n' + source.read_text())
|
||||
monkeypatch.setattr(_sandbox, "run", lambda *a, **kw: pytest.fail("task entry attempted namespaces"))
|
||||
result = verify_side_task(sides[0], demo, "path")
|
||||
assert result.outcome == "observed", result.reason
|
||||
assert len(result.runs) == 1 and result.runs[0].observed
|
||||
assert result.isolation_mode == "task-boundary"
|
||||
|
||||
def changed(job, **kwargs):
|
||||
result = original(job, **kwargs)
|
||||
if job.command[-1] == "/demo/build.sh":
|
||||
(kwargs["mounts"]["/deps/0"] / "module.py").write_text("replacement")
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(_sandbox, "run", changed)
|
||||
def test_capture_canary_cannot_replay(pair):
|
||||
sides, demo = pair
|
||||
edit(demo, lambda value: value["steps"][0].update(capture={"name": "stolen", "source": "output"}))
|
||||
result = verify(*sides, demo, "path")
|
||||
assert result.outcome == "inconclusive"
|
||||
assert result.sides[0].reason == "build changed immutable inputs"
|
||||
assert result.outcome != "demonstrated"
|
||||
assert "canary values cannot be captured" in result.sides[0].reason
|
||||
|
||||
|
||||
def test_trusted_build_products_are_used_readonly(pair):
|
||||
def test_ordered_cli_capture_and_stdin(pair):
|
||||
sides, demo = pair
|
||||
sides = [
|
||||
Side(s.checkout, ("/usr/bin/python3", "-I", "/build/app.py"), build_command=("/bin/cp", "/workspace/app.py", "/scratch/app.py"))
|
||||
for s in sides
|
||||
]
|
||||
request(demo, "path")
|
||||
app = """import json, os, sys
|
||||
if sys.argv[1:] == ['--ready']: sys.exit(0)
|
||||
if sys.argv[1:] == ['token']: print(json.dumps({'token':'public-token'}))
|
||||
elif sys.argv[1:] == ['use', 'public-token'] and sys.stdin.read() == 'public-token':
|
||||
from pathlib import Path
|
||||
if not (Path(__file__).parent / 'fixed').exists(): print((Path(os.environ['SERVED_ROOT']).parent / 'canary').read_text())
|
||||
"""
|
||||
for side in sides:
|
||||
(side.checkout / "app.py").write_text(app)
|
||||
edit(
|
||||
demo,
|
||||
lambda value: value.update(
|
||||
steps=[
|
||||
{"type": "cli", "arguments": ["token"], "capture": {"name": "token", "source": "json", "path": ["token"]}},
|
||||
{"type": "cli", "arguments": ["use", "${token}"], "stdin": "${token}"},
|
||||
]
|
||||
),
|
||||
)
|
||||
assert verify(*sides, demo, "path").outcome == "demonstrated"
|
||||
(demo / "build.sh").write_text("echo patched > /build/app.py")
|
||||
assert verify(*sides, demo, "path").outcome == "inconclusive"
|
||||
|
||||
|
||||
def test_http_steps_and_capture(pair):
|
||||
try:
|
||||
with socket.socket():
|
||||
pass
|
||||
except PermissionError:
|
||||
pytest.skip("host policy prohibits localhost sockets")
|
||||
sides, demo = pair
|
||||
app = """from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from pathlib import Path
|
||||
import json, os
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200); self.end_headers()
|
||||
if self.path == '/token': result=json.dumps({'token':'public-token'})
|
||||
elif self.path == '/use/public-token' and not (Path(__file__).parent/'fixed').exists():
|
||||
result=(Path(os.environ['SERVED_ROOT']).parent/'canary').read_text()
|
||||
else: result='safe'
|
||||
self.wfile.write(result.encode())
|
||||
def log_message(self, *args): pass
|
||||
HTTPServer(('127.0.0.1', 18765), Handler).serve_forever()
|
||||
"""
|
||||
for side in sides:
|
||||
(side.checkout / "app.py").write_text(app)
|
||||
|
||||
def change(value):
|
||||
value["start"].update(mode="http", port=18765)
|
||||
value["steps"] = [
|
||||
{"type": "http", "method": "GET", "path": "/token", "capture": {"name": "token", "source": "json", "path": ["token"]}},
|
||||
{"type": "http", "method": "GET", "path": "/use/${token}"},
|
||||
]
|
||||
|
||||
edit(demo, change)
|
||||
result = verify(*sides, demo, "path")
|
||||
assert result.outcome == "demonstrated", result
|
||||
|
||||
|
||||
def test_private_namespace_handoff_without_sockets(tmp_path):
|
||||
@@ -351,19 +262,6 @@ def test_private_namespace_handoff_without_sockets(tmp_path):
|
||||
process.stdout.close()
|
||||
|
||||
|
||||
def test_attacker_sees_only_external_bridge_and_scratch(pair):
|
||||
sides, demo = pair
|
||||
(demo / "attack.sh").write_text(
|
||||
"test ! -e /workspace/app.py || exit 3\n"
|
||||
"test ! -e /fixture/canary || exit 3\n"
|
||||
"test ! -e /build/app.py || exit 3\n"
|
||||
"test ! -e /deps/0 || exit 3\n"
|
||||
'test -z "$CALLBACK_URL$PROOF_MARKER$DATABASE$SERVED_ROOT" || exit 3\n'
|
||||
'"$TARGET" path ../canary\n'
|
||||
)
|
||||
assert verify(*sides, demo, "path").outcome == "demonstrated"
|
||||
|
||||
|
||||
def test_listener_accepts_nonce_path_for_post_without_sockets(monkeypatch, capsys):
|
||||
import fcntl
|
||||
import http.server
|
||||
@@ -409,3 +307,168 @@ def test_listener_accepts_nonce_path_for_post_without_sockets(monkeypatch, capsy
|
||||
monkeypatch.setattr(http.server, "HTTPServer", Server)
|
||||
oracles._listen("secret")
|
||||
assert capsys.readouterr().out.splitlines()[1:] == ["observed"]
|
||||
|
||||
|
||||
def test_pip_products_imported_by_runtime_and_immutable(pair, monkeypatch):
|
||||
from openultrasast.sandbox import SandboxResult
|
||||
|
||||
sides, demo = pair
|
||||
for side in sides:
|
||||
source = side.checkout / "app.py"
|
||||
source.write_text("import dependency\nassert dependency.VALUE == 42\n" + source.read_text())
|
||||
(side.checkout / "requirements.txt").write_text("dependency==1\n")
|
||||
edit(demo, lambda value: value["build"].update(recipe="pip", arguments=["requirements.txt"]))
|
||||
original = _sandbox.run
|
||||
builds = []
|
||||
|
||||
def runner(job, **kwargs):
|
||||
if "pip" in job.command:
|
||||
assert "--no-index" in job.command and "--no-deps" in job.command
|
||||
packages = kwargs["scratch"] / "packages"
|
||||
packages.mkdir()
|
||||
(packages / "dependency.py").write_text("VALUE = 42\n")
|
||||
builds.append(job)
|
||||
return SandboxResult(0, "", "", False)
|
||||
return original(job, **kwargs)
|
||||
|
||||
monkeypatch.setattr(_sandbox, "run", runner)
|
||||
result = verify(*sides, demo, "path")
|
||||
assert result.outcome == "demonstrated", result
|
||||
assert len(builds) == 6
|
||||
|
||||
|
||||
@pytest.mark.parametrize("count", [0, 1, 2, 4])
|
||||
def test_dispatcher_incomplete_repetitions_never_demonstrate(pair, monkeypatch, count):
|
||||
sides, demo = pair
|
||||
|
||||
def unavailable(**kwargs):
|
||||
raise _sandbox.IsolationUnavailable("denied")
|
||||
|
||||
monkeypatch.setattr(_sandbox, "isolation_check", unavailable)
|
||||
|
||||
def dispatch(**kwargs):
|
||||
return SideRecord("observed", (RunObservation(kwargs["side"] == sides[0], "", 0),) * count, 0, "", isolation_mode="task-boundary")
|
||||
|
||||
assert verify(*sides, demo, "path", task_dispatcher=dispatch).outcome == "inconclusive"
|
||||
|
||||
|
||||
def test_json_file_accepted_without_directory_wrapper(pair):
|
||||
sides, demo = pair
|
||||
assert verify(*sides, demo / "demo.json", "path").outcome == "demonstrated"
|
||||
|
||||
|
||||
def test_http_startup_effect_vetoed_before_request_steps(tmp_path):
|
||||
from openultrasast.sandbox import SandboxJob, SandboxResult
|
||||
from openultrasast.search.verify import _http_steps
|
||||
|
||||
checks = []
|
||||
|
||||
def runner(job, **kwargs):
|
||||
config = json.loads(kwargs["input_text"])
|
||||
(kwargs["scratch"] / config["ready"]).touch()
|
||||
# A startup effect must veto the go signal; no requests get authorized.
|
||||
import time
|
||||
|
||||
time.sleep(0.05)
|
||||
assert not (kwargs["scratch"] / config["go"]).exists()
|
||||
return SandboxResult(1, "", "not authorized", False)
|
||||
|
||||
def check():
|
||||
checks.append(True)
|
||||
raise ValueError("startup effect")
|
||||
|
||||
with pytest.raises(ValueError, match="startup effect"):
|
||||
_http_steps(
|
||||
{}, (), lambda command: SandboxJob("", command, tmp_path, {}, 1, 256, 128), tmp_path, {}, {}, None, "nonce", runner, check
|
||||
)
|
||||
assert checks == [True]
|
||||
|
||||
|
||||
def test_explicit_task_dispatcher_used_on_userns_host(pair, monkeypatch):
|
||||
sides, demo = pair
|
||||
monkeypatch.setattr(_sandbox, "isolation_check", lambda **kwargs: pytest.fail("remote verification must not probe local execution"))
|
||||
calls = []
|
||||
|
||||
def dispatch(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return SideRecord("observed", (RunObservation(kwargs["side"] == sides[0], "", 0),) * 3, 0, "", isolation_mode="task-boundary")
|
||||
|
||||
result = verify(*sides, demo, "path", task_dispatcher=dispatch)
|
||||
assert result.outcome == "demonstrated" and len(calls) == 2
|
||||
assert result.isolation_mode == "task-boundary"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("handshake", [False, True])
|
||||
def test_http_driver_requires_readiness_and_complete_outputs(tmp_path, handshake):
|
||||
from openultrasast.sandbox import SandboxJob, SandboxResult
|
||||
from openultrasast.search.verify import _http_steps
|
||||
|
||||
def runner(job, **kwargs):
|
||||
config = json.loads(kwargs["input_text"])
|
||||
if handshake:
|
||||
(kwargs["scratch"] / config["ready"]).touch()
|
||||
return SandboxResult(0, "[]", "", False)
|
||||
|
||||
with pytest.raises(ValueError, match="readiness|invalid HTTP driver output"):
|
||||
_http_steps(
|
||||
{"steps": [{}]},
|
||||
(),
|
||||
lambda command: SandboxJob("", command, tmp_path, {}, 1, 256, 128),
|
||||
tmp_path,
|
||||
{},
|
||||
{},
|
||||
None,
|
||||
"nonce",
|
||||
runner,
|
||||
lambda: None,
|
||||
)
|
||||
|
||||
|
||||
def test_generated_build_links_confined_and_hashed(tmp_path):
|
||||
from openultrasast.search.verify import _tree
|
||||
|
||||
checkout = tmp_path / "checkout"
|
||||
checkout.mkdir()
|
||||
module = checkout / "node_modules" / "package"
|
||||
module.mkdir(parents=True)
|
||||
(module / "cli.js").write_text("original")
|
||||
bins = checkout / "node_modules" / ".bin"
|
||||
bins.mkdir()
|
||||
(bins / "cli").symlink_to("../package/cli.js")
|
||||
initial = _tree(checkout, generated_links=True)
|
||||
with pytest.raises(ValueError, match="links"):
|
||||
_tree(checkout)
|
||||
(module / "cli.js").write_text("changed")
|
||||
assert _tree(checkout, generated_links=True) != initial
|
||||
(bins / "escape").symlink_to("/etc/passwd")
|
||||
with pytest.raises(ValueError, match="escapes"):
|
||||
_tree(checkout, generated_links=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", ["missing", "link", "."])
|
||||
def test_generated_dangling_and_cyclic_links_rejected(tmp_path, target):
|
||||
from openultrasast.search.verify import _tree
|
||||
|
||||
(tmp_path / "link").symlink_to(target)
|
||||
with pytest.raises(ValueError, match="dangling|cyclic"):
|
||||
_tree(tmp_path, generated_links=True)
|
||||
|
||||
|
||||
def test_http_failure_keeps_exception_tail(tmp_path):
|
||||
from openultrasast.sandbox import SandboxJob, SandboxResult
|
||||
from openultrasast.search.verify import _http_steps
|
||||
|
||||
traceback = "Traceback (most recent call last):\n" + " frame\n" * 200 + "FileNotFoundError: /dev/null\n"
|
||||
with pytest.raises(ValueError, match="FileNotFoundError: /dev/null"):
|
||||
_http_steps(
|
||||
{},
|
||||
("/bin/true",),
|
||||
lambda command: SandboxJob("", command, tmp_path, {}, 1, 256, 128),
|
||||
tmp_path,
|
||||
{},
|
||||
{},
|
||||
None,
|
||||
"nonce",
|
||||
lambda *a, **kw: SandboxResult(1, "", traceback, False),
|
||||
lambda: None,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tarfile
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from benchmarks.ax import search_verify as transport
|
||||
from openultrasast.search import verify_task as entry
|
||||
from openultrasast.search.verify import RunObservation, Side, SideRecord
|
||||
|
||||
|
||||
def demo():
|
||||
return {
|
||||
"build": {"recipe": "none", "arguments": []},
|
||||
"start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
|
||||
"steps": [{"type": "cli", "arguments": []}],
|
||||
}
|
||||
|
||||
|
||||
def record():
|
||||
return SideRecord("observed", (RunObservation(True, "proof", 0.01),), 0.1, "observed", isolation_mode="task-boundary")
|
||||
|
||||
|
||||
def test_dispatch_uses_fresh_tasks_and_only_data(tmp_path):
|
||||
root = tmp_path / "repo"
|
||||
root.mkdir()
|
||||
(root / "app.py").write_text("print(1)")
|
||||
calls = []
|
||||
|
||||
def lane(item, output, attempt):
|
||||
calls.append(item)
|
||||
assert item.extra_env == {}
|
||||
assert set(item.inputs) == {"VERIFY_INPUT_URL"}
|
||||
assert item.inputs["VERIFY_INPUT_URL"] == b"executor-built-archive"
|
||||
return asdict(record())
|
||||
|
||||
dispatch = transport.AXSideDispatcher(
|
||||
SimpleNamespace(), "image@sha256:" + "a" * 64, tmp_path / "out", lane=lane, prepare=lambda side, spec: b"executor-built-archive"
|
||||
)
|
||||
result = dispatch(side=Side(root), demo=demo(), family="sql", timeout_seconds=2, fresh_task=True)
|
||||
assert result.isolation_mode == "task-boundary"
|
||||
assert len(result.runs) == 3
|
||||
assert len(calls) == 3 and calls[0].id != calls[1].id
|
||||
assert all(i.id.startswith("search-verify-") for i in calls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name,kind", [("../escape", tarfile.REGTYPE), ("/escape", tarfile.REGTYPE), ("app.py", tarfile.SYMTYPE), ("app.py", tarfile.LNKTYPE)]
|
||||
)
|
||||
def test_extract_rejects_escape_and_links(tmp_path, name, kind):
|
||||
data = io.BytesIO()
|
||||
with tarfile.open(fileobj=data, mode="w:gz") as archive:
|
||||
member = tarfile.TarInfo(name)
|
||||
member.type = kind
|
||||
member.linkname = "/etc/passwd"
|
||||
archive.addfile(member)
|
||||
with pytest.raises(ValueError):
|
||||
entry.extract_checkout(data.getvalue(), tmp_path / "repo")
|
||||
|
||||
|
||||
def test_entry_reads_input_and_scrubs_environment(tmp_path, monkeypatch, capsys):
|
||||
root = tmp_path / "repo"
|
||||
root.mkdir()
|
||||
(root / "app.py").write_text("print(1)")
|
||||
bundle = tmp_path / "bundle"
|
||||
bundle.mkdir()
|
||||
import shutil
|
||||
|
||||
shutil.copytree(root, bundle / "checkout")
|
||||
(bundle / "products").mkdir()
|
||||
(bundle / "spec.json").write_text(json.dumps({"demo": demo(), "family": "sql", "timeout_seconds": 2}))
|
||||
blob = entry.pack_checkout(bundle)
|
||||
monkeypatch.setattr(entry, "download", lambda url, limit: blob)
|
||||
outputs = []
|
||||
monkeypatch.setattr(entry, "upload", lambda url, data: outputs.append(json.loads(data)))
|
||||
|
||||
def execute(side, schema, family, timeout_seconds):
|
||||
assert (side.checkout / "app.py").read_text() == "print(1)"
|
||||
assert "HOME" not in entry.os.environ
|
||||
return record()
|
||||
|
||||
monkeypatch.setattr(entry, "run_side", execute)
|
||||
monkeypatch.setattr(
|
||||
entry.os,
|
||||
"environ",
|
||||
{
|
||||
"VERIFY_INPUT_URL": "https://store.example/input",
|
||||
"RESULT_URL": "https://store.example/result",
|
||||
},
|
||||
)
|
||||
entry.task_main()
|
||||
assert outputs[0]["outcome"] == "observed"
|
||||
assert "checkout_bytes=" in capsys.readouterr().out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", ["OPENAI_API_KEY", "OPENAI_KEY", "GEMINI_TOKEN", "DEEPSEEK_API_KEY"])
|
||||
def test_entry_rejects_inherited_model_key_before_download(monkeypatch, key):
|
||||
monkeypatch.setattr(entry.os, "environ", {key: "secret"})
|
||||
monkeypatch.setattr(entry, "download", lambda *a: pytest.fail("must not read input"))
|
||||
with pytest.raises(ValueError, match="credentials"):
|
||||
entry.task_main()
|
||||
|
||||
|
||||
def test_archive_expansion_limit(tmp_path, monkeypatch):
|
||||
root = tmp_path / "source"
|
||||
root.mkdir()
|
||||
(root / "app.py").write_bytes(b"A" * 1024)
|
||||
data = entry.pack_checkout(root)
|
||||
monkeypatch.setattr(entry, "MAX_CHECKOUT_BYTES", 10)
|
||||
with pytest.raises(ValueError, match="expanded checkout"):
|
||||
entry.extract_checkout(data, tmp_path / "destination")
|
||||
|
||||
|
||||
def test_pack_refuses_symlinks(tmp_path):
|
||||
(tmp_path / "outside").symlink_to("/etc/passwd")
|
||||
with pytest.raises(ValueError, match="special files"):
|
||||
entry.pack_checkout(tmp_path)
|
||||
|
||||
|
||||
def test_result_requires_single_completed_attested_run(tmp_path):
|
||||
payload = asdict(record())
|
||||
for change in ({"isolation_mode": "userns"}, {"runs": []}, {"runs": [payload["runs"][0]] * 3}):
|
||||
with pytest.raises(ValueError):
|
||||
transport.validate_result(json.dumps({**payload, **change}).encode(), tmp_path)
|
||||
|
||||
|
||||
def test_production_item_targets_packaged_module(tmp_path):
|
||||
root = tmp_path / "repo"
|
||||
root.mkdir()
|
||||
(root / "app.py").write_text("print(1)")
|
||||
captured = []
|
||||
|
||||
class Lane:
|
||||
def __init__(self, args, workload):
|
||||
captured.append(workload)
|
||||
|
||||
def __call__(self, item, output, attempt):
|
||||
assert item.command == ("python3", "-m", "openultrasast.search.verify_task")
|
||||
transport.validate_result(json.dumps(asdict(record())).encode(), output)
|
||||
return json.loads((output / "result.json").read_text())
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch.object(transport, "AXLane", Lane):
|
||||
dispatcher = transport.AXSideDispatcher(
|
||||
SimpleNamespace(), "image@sha256:" + "a" * 64, tmp_path / "out", prepare=lambda side, spec: b"built"
|
||||
)
|
||||
dispatcher(side=Side(root), demo=demo(), family="sql", timeout_seconds=2, fresh_task=True)
|
||||
assert captured[0].url_env == frozenset({"VERIFY_INPUT_URL", "RESULT_URL"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outcome", ["could_not_build", "could_not_run", "no_oracle", "inconclusive"])
|
||||
def test_incomplete_outcomes_survive_transport(tmp_path, outcome):
|
||||
payload = {**asdict(record()), "outcome": outcome, "runs": []}
|
||||
transport.validate_result(json.dumps(payload).encode(), tmp_path)
|
||||
assert json.loads((tmp_path / "result.json").read_text())["outcome"] == outcome
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [-1, float("nan"), float("inf"), "0", True])
|
||||
def test_result_rejects_invalid_measurements(tmp_path, value):
|
||||
with pytest.raises(ValueError):
|
||||
transport.validate_result(json.dumps({**asdict(record()), "elapsed_seconds": value}).encode(), tmp_path)
|
||||
|
||||
|
||||
def test_default_dispatch_builds_in_executor_before_verifier(tmp_path, monkeypatch):
|
||||
root = tmp_path / "repo"
|
||||
root.mkdir()
|
||||
(root / "app.py").write_text("print('input')")
|
||||
events = []
|
||||
|
||||
class Executor:
|
||||
def __init__(self, lane, archive):
|
||||
assert archive
|
||||
|
||||
def __enter__(self):
|
||||
events.append("executor-start")
|
||||
return self
|
||||
|
||||
def prepare(self, spec):
|
||||
assert spec["demo"] == demo()
|
||||
events.append("build")
|
||||
return b"built"
|
||||
|
||||
def __exit__(self, *args):
|
||||
events.append("executor-deleted")
|
||||
|
||||
def lane(item, output, attempt):
|
||||
assert events[:3] == ["executor-start", "build", "executor-deleted"]
|
||||
assert item.inputs == {"VERIFY_INPUT_URL": b"built"}
|
||||
events.append("verify")
|
||||
return asdict(record())
|
||||
|
||||
monkeypatch.setattr(transport, "SearchExecutorTask", Executor)
|
||||
dispatch = transport.AXSideDispatcher(SimpleNamespace(), "image@sha256:" + "a" * 64, tmp_path / "out", lane=lane)
|
||||
assert len(dispatch(side=Side(root), demo=demo(), family="sql", timeout_seconds=2, fresh_task=True).runs) == 3
|
||||
assert events[3:] == ["verify"] * 3
|
||||
|
||||
|
||||
def test_source_archive_omits_git_metadata(tmp_path):
|
||||
root = tmp_path / "repo"
|
||||
root.mkdir()
|
||||
(root / "app.py").write_text("print('source')")
|
||||
(root / ".git").write_text("gitdir: /host/private/worktree")
|
||||
archive = entry.pack_checkout(root)
|
||||
target = tmp_path / "extracted"
|
||||
from openultrasast.search.executor import unpack_checkout
|
||||
|
||||
unpack_checkout(archive, target)
|
||||
assert (target / "app.py").read_text() == "print('source')"
|
||||
assert not (target / ".git").exists()
|
||||
@@ -8,6 +8,7 @@ from openultrasast.search import _sandbox
|
||||
from openultrasast.search.board import Board
|
||||
from openultrasast.search.budget import SearchBudget
|
||||
from openultrasast.search.coordinator import Coordinator, SearchTask
|
||||
from openultrasast.search.executor import InProcessExecutor
|
||||
from openultrasast.search.probe import materialise
|
||||
from openultrasast.search.verify import verify
|
||||
from openultrasast.search.worker import StubModel, Worker
|
||||
@@ -30,7 +31,7 @@ def test_reason_no_tools_and_bounded(tmp_path):
|
||||
result = Worker(model, repo=tmp_path, demo=tmp_path / "demo")(task())
|
||||
assert result["intents"][0]["description"] == "inspect"
|
||||
assert model.calls[0]["tools"] == []
|
||||
assert result["isolation_mode"] == _sandbox.isolation_mode()
|
||||
assert result["isolation_mode"] == "brain-only"
|
||||
assert result["cost_usd"] == 0
|
||||
model = StubModel([reply({"intents": [{"description": "x"}] * 4})])
|
||||
assert Worker(model, repo=tmp_path, demo=tmp_path / "demo")(task())["status"] == "execution_failure"
|
||||
@@ -41,7 +42,7 @@ def test_tools_bounds_and_path_escape(tmp_path):
|
||||
repo.mkdir()
|
||||
(repo / "many").write_text("needle\n" * 1000)
|
||||
(repo / "link").symlink_to("/etc/passwd")
|
||||
worker = Worker(StubModel([]), repo=repo, demo=tmp_path / "demo")
|
||||
worker = Worker(StubModel([]), repo=repo, demo=tmp_path / "demo", executor=InProcessExecutor(repo))
|
||||
assert len(worker.tool("read_file", {"path": "many", "lines": 900})["text"].splitlines()) == 400
|
||||
assert len(worker.tool("grep", {"pattern": "needle"})["matches"]) == 200
|
||||
for path in ("../escape", "/etc/passwd", "link"):
|
||||
@@ -53,7 +54,7 @@ def test_tools_bounds_and_path_escape(tmp_path):
|
||||
|
||||
def test_run_scrubs_model_key_in_real_sandbox(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "must-not-reach-repo")
|
||||
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo")
|
||||
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path))
|
||||
result = worker.tool(
|
||||
"run",
|
||||
{
|
||||
@@ -68,18 +69,17 @@ def test_run_scrubs_model_key_in_real_sandbox(tmp_path, monkeypatch):
|
||||
|
||||
|
||||
def test_coordinator_stub_smoke_real_verify(tmp_path):
|
||||
sides, _ = materialise(tmp_path / "pair", "path")
|
||||
sides, probe_demos = materialise(tmp_path / "pair", "path")
|
||||
demo = tmp_path / "demo"
|
||||
model = StubModel(
|
||||
[
|
||||
reply({"intents": [{"description": "prove path traversal", "from_facts": []}]}),
|
||||
tool("read_file", path="app.py"),
|
||||
tool("write_demo", path="build.sh", content="test -s /workspace/app.py\n"),
|
||||
tool("write_demo", path="request.json", content='["path", "../canary"]'),
|
||||
tool("write_demo", schema=json.loads((probe_demos["real"] / "demo.json").read_text())),
|
||||
tool("finish", fact={"text": "Traversal candidate", "evidence_refs": ["tool:1"], "demo": "demo/"}),
|
||||
]
|
||||
)
|
||||
worker = Worker(model, repo=sides[0].checkout, demo=demo)
|
||||
worker = Worker(model, repo=sides[0].checkout, demo=demo, executor=InProcessExecutor(sides[0].checkout))
|
||||
outcomes = []
|
||||
|
||||
def executor(job):
|
||||
@@ -131,7 +131,7 @@ def test_failed_model_call_charged_and_secrets_not_reported(tmp_path):
|
||||
|
||||
|
||||
def test_run_timeout_truncation_and_disk_bound(tmp_path):
|
||||
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo")
|
||||
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path))
|
||||
result = worker.tool("run", {"command": ["/bin/sleep", "3"], "timeout_seconds": 1})
|
||||
assert result["timed_out"]
|
||||
result = worker.tool("run", {"command": ["/usr/bin/python3", "-I", "-c", 'print("x" * 20000)']})
|
||||
@@ -159,3 +159,73 @@ def test_reason_completion_cites_board_facts(tmp_path):
|
||||
from dataclasses import replace
|
||||
|
||||
assert worker(replace(job, snapshot="facts: [{id: f1}]"))["complete"] == good["complete"]
|
||||
|
||||
|
||||
def test_brain_routes_every_repository_tool_without_local_access(tmp_path, monkeypatch):
|
||||
class RecordingExecutor:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def submit(self, name, args, **kwargs):
|
||||
self.calls.append((name, args, kwargs))
|
||||
return {"executor": name}
|
||||
|
||||
executor = RecordingExecutor()
|
||||
worker = Worker(StubModel([]), repo=tmp_path / "absent", demo=tmp_path / "demo", executor=executor)
|
||||
import subprocess
|
||||
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError("brain touched repository")
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", forbidden)
|
||||
monkeypatch.setattr(type(tmp_path), "open", forbidden)
|
||||
for name, args in (
|
||||
("list_files", {}),
|
||||
("read_file", {"path": "app.py"}),
|
||||
("grep", {"pattern": "needle"}),
|
||||
("run", {"command": ["python3", "app.py"]}),
|
||||
):
|
||||
assert worker.tool(name, args) == {"executor": name}
|
||||
assert [c[0] for c in executor.calls] == ["list_files", "read_file", "grep", "run"]
|
||||
|
||||
|
||||
def test_brain_refuses_repository_tools_without_executor(tmp_path):
|
||||
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo")
|
||||
with pytest.raises(ValueError, match="executor"):
|
||||
worker.tool("list_files", {})
|
||||
|
||||
|
||||
def test_write_demo_accepts_only_declarative_json(tmp_path):
|
||||
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo")
|
||||
schema = {
|
||||
"build": {"recipe": "none", "arguments": []},
|
||||
"start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
|
||||
"steps": [{"type": "cli", "arguments": ["hello"]}],
|
||||
}
|
||||
assert worker.tool("write_demo", {"schema": schema})["path"] == "demo/demo.json"
|
||||
assert json.loads((tmp_path / "demo/demo.json").read_text()) == schema
|
||||
with pytest.raises(ValueError):
|
||||
worker.tool("write_demo", {"path": "build.sh", "content": "touch /tmp/pwn"})
|
||||
|
||||
|
||||
def test_inprocess_executor_observes_all_brain_repository_commands(tmp_path):
|
||||
class RecordingExecutor(InProcessExecutor):
|
||||
def __init__(self, repo):
|
||||
super().__init__(repo)
|
||||
self.commands = []
|
||||
|
||||
def execute(self, command):
|
||||
self.commands.append(command)
|
||||
return super().execute(command)
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "sample.txt").write_text("needle")
|
||||
executor = RecordingExecutor(repo)
|
||||
worker = Worker(StubModel([]), repo=repo, demo=tmp_path / "demo", executor=executor)
|
||||
assert worker.tool("list_files", {})["files"] == ["sample.txt"]
|
||||
assert worker.tool("read_file", {"path": "sample.txt"})["input_bytes"] == 6
|
||||
assert worker.tool("grep", {"pattern": "needle"})["matches"][0]["line"] == 1
|
||||
assert worker.tool("run", {"command": ["/bin/cat", "/workspace/sample.txt"]})["stdout"] == "needle"
|
||||
assert [c.name for c in executor.commands] == ["list_files", "read_file", "grep", "run"]
|
||||
assert [c.seq for c in executor.commands] == [1, 2, 3, 4]
|
||||
|
||||
Reference in New Issue
Block a user