mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5.5
85f4f0b3d4
Demos are data the verifier executes (fixed build recipes, runtime+path start, HTTP/CLI steps with captures; the canary is never referable). The agent loop with the key never runs repository code: executor tasks without a key exchange commands and results through presigned objects. AX capacity errors are back-pressure with at most 2 tasks in flight; caches and builds under /workspace with a 2 GiB guard; verifiers take one presigned archive per side. Launcher fix: private /dev in bwrap. 190 tests on the host; probe 5/5 demonstrated. Co-Authored-By: Claude Opus 5.5 <[email protected]>
121 lines
4.5 KiB
Python
121 lines
4.5 KiB
Python
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from openultrasast.search import probe
|
|
from openultrasast.search._sandbox import IsolationUnavailable
|
|
|
|
|
|
def test_isolation_failure_is_recorded_and_nonzero(monkeypatch, capsys):
|
|
def fail(**kwargs):
|
|
raise IsolationUnavailable("gVisor denied namespace")
|
|
|
|
monkeypatch.setattr(probe._sandbox, "isolation_check", fail)
|
|
monkeypatch.setattr(probe, "image_facts", lambda: {})
|
|
monkeypatch.delenv("RESULT_URL", raising=False)
|
|
assert probe.main([]) == 1
|
|
record = json.loads(capsys.readouterr().out)
|
|
assert record["status"] == "instrument_failure"
|
|
assert "gVisor denied namespace" in record["exception"]
|
|
assert record["oracles"] == {}
|
|
|
|
|
|
def test_packaged_pairs_and_three_demos(tmp_path):
|
|
for family in probe.FAMILIES:
|
|
sides, demos = probe.materialise(tmp_path / family, family)
|
|
assert len(demos) == 3
|
|
assert all((s.checkout / "app.py").stat().st_size > 0 for s in sides)
|
|
assert (sides[1].checkout / "fixed").exists()
|
|
assert json.loads((demos["real"] / "demo.json").read_text())["steps"][0]["arguments"][0] == family
|
|
|
|
|
|
def test_batch_workload_and_all_lanes(tmp_path, monkeypatch):
|
|
from types import SimpleNamespace
|
|
|
|
from benchmarks.ax import search_probe as adapter
|
|
from benchmarks.ax.batch import manifest
|
|
|
|
image = "registry/search@sha256:" + "a" * 64
|
|
item = adapter.probe_item(image)
|
|
workload = adapter.probe_workload(image)
|
|
doc = manifest(
|
|
"ousast-engine-search-probe-123456789012", workload, item, {"RESULT_URL": "https://objects.example.org/probe"}, "default"
|
|
)
|
|
assert doc["spec"]["command"] == ["python3", "-m", "openultrasast.search.probe"]
|
|
assert item.inputs == {}
|
|
called = []
|
|
|
|
def lane(args, workload):
|
|
called.append(args.lane)
|
|
return lambda *args: {"status": "ok"}
|
|
|
|
monkeypatch.setattr(adapter, "AXLane", lane)
|
|
monkeypatch.setattr(adapter, "DockerLane", lane)
|
|
for name in ("ax", "kind", "docker"):
|
|
args = SimpleNamespace(image=image, lane=name, deadline=900, out=tmp_path / name)
|
|
assert not adapter.run_probe(args)
|
|
assert called == ["ax", "kind", "docker"]
|
|
|
|
|
|
def test_probe_runs_all_fifteen_verifications(tmp_path, monkeypatch):
|
|
from openultrasast.search.verify import SideRecord, VerificationRecord
|
|
|
|
calls = []
|
|
monkeypatch.setattr(probe._sandbox, "isolation_check", lambda: None)
|
|
monkeypatch.setattr(probe, "image_facts", lambda: {"python": {"version": "test"}})
|
|
monkeypatch.setattr(probe, "BrowserExecutor", lambda: object())
|
|
|
|
def verify(a, b, demo, family, **kwargs):
|
|
calls.append((family, demo.name))
|
|
return VerificationRecord(
|
|
"demonstrated" if demo.name == "real" else "inconclusive", (SideRecord("observed", (), 1, "", 0.2, 0.3, 0.5),) * 2, 2, ""
|
|
)
|
|
|
|
monkeypatch.setattr(probe, "verify", verify)
|
|
record = probe.probe(tmp_path)
|
|
assert record["status"] == "ok"
|
|
assert len(calls) == 15
|
|
assert record["oracles"]["sql"]["real"]["sides"][0]["ready_seconds"] == 0.3
|
|
assert record["scratch_disk_bytes"] > 0
|
|
|
|
|
|
def test_result_put_same_record_without_credentials(monkeypatch, capsys):
|
|
from io import BytesIO
|
|
|
|
monkeypatch.setenv("RESULT_URL", "https://store.example/probe?signature=secret")
|
|
monkeypatch.setattr(probe, "probe", lambda root: {"status": "ok"})
|
|
seen = []
|
|
|
|
class Response(BytesIO):
|
|
status = 200
|
|
|
|
def upload(request, **kwargs):
|
|
seen.append(request)
|
|
return Response()
|
|
|
|
monkeypatch.setattr(probe.urllib.request, "urlopen", upload)
|
|
assert probe.main([]) == 0
|
|
assert seen[0].data == capsys.readouterr().out.encode()
|
|
assert seen[0].method == "PUT"
|
|
assert not seen[0].has_header("Authorization")
|
|
|
|
|
|
def test_oracle_namespace_failure_is_instrument_failure(tmp_path, monkeypatch):
|
|
from openultrasast.search.verify import SideRecord, VerificationRecord
|
|
|
|
monkeypatch.setattr(probe._sandbox, "isolation_check", lambda: None)
|
|
monkeypatch.setattr(probe, "image_facts", lambda: {})
|
|
monkeypatch.setattr(
|
|
probe,
|
|
"verify",
|
|
lambda *a, **kw: VerificationRecord(
|
|
"inconclusive", (SideRecord("could_not_run", (), 0, "verification unavailable or refused: net namespace denied"),), 0, ""
|
|
),
|
|
)
|
|
record = probe.probe(tmp_path)
|
|
assert record["status"] == "instrument_failure"
|
|
assert "net namespace denied" in record["exception"]
|
|
assert record["isolation_check"]["status"] == "ok"
|