mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
search-with-proof: probe harness (3.2) and agent worker (group 5)
Probe runs each oracle toy pair with a real demo and two gaming demos, records image facts, RSS and disk, fails loudly if isolation is unavailable; dispatchable as an AX task. Worker: reason step without tools, explore tool loop whose run tool goes through the sandbox with a scrubbed env (model key absent), spend reserved before each call. Host probe: 5/5 real demos demonstrated, 10/10 gaming demos inconclusive. 89 tests. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
a17a6afaf8
commit
0f74767db9
Vendored
-33
@@ -1,33 +0,0 @@
|
||||
"""CLI toy: arguments are its sole public interface; config belongs to the app."""
|
||||
|
||||
import os
|
||||
import pathlib
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
fixed = pathlib.Path("/workspace/fixed").exists()
|
||||
if sys.argv[1:] == ["--ready"]:
|
||||
sys.exit(1 if pathlib.Path("/workspace/broken").exists() else 0)
|
||||
family, payload = sys.argv[1:]
|
||||
if family == "path":
|
||||
root = pathlib.Path(os.environ["SERVED_ROOT"])
|
||||
path = (root / payload).resolve()
|
||||
if not fixed or path.is_relative_to(root):
|
||||
value = path.read_text()
|
||||
if not pathlib.Path("/workspace/flaky").exists() or value.startswith("a"):
|
||||
print(value)
|
||||
elif family == "sql":
|
||||
with sqlite3.connect(os.environ["DATABASE"]) as db:
|
||||
if fixed:
|
||||
rows = db.execute("SELECT value FROM records WHERE public=1 AND name=?", (payload,))
|
||||
else:
|
||||
rows = db.execute("SELECT value FROM records WHERE public=1 AND name='" + payload + "'")
|
||||
print(list(rows))
|
||||
elif family == "command":
|
||||
if not fixed:
|
||||
subprocess.run("echo " + payload, shell=True, check=False)
|
||||
elif family == "ssrf":
|
||||
if not fixed:
|
||||
print(urllib.request.urlopen(os.path.expandvars(payload), timeout=2).read())
|
||||
@@ -0,0 +1,120 @@
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.search import probe
|
||||
from openultrasast.search._sandbox import IsolationUnavailable
|
||||
|
||||
|
||||
def test_isolation_failure_is_recorded_and_nonzero(monkeypatch, capsys):
|
||||
def fail(**kwargs):
|
||||
raise IsolationUnavailable("gVisor denied namespace")
|
||||
|
||||
monkeypatch.setattr(probe._sandbox, "isolation_check", fail)
|
||||
monkeypatch.setattr(probe, "image_facts", lambda: {})
|
||||
monkeypatch.delenv("RESULT_URL", raising=False)
|
||||
assert probe.main([]) == 1
|
||||
record = json.loads(capsys.readouterr().out)
|
||||
assert record["status"] == "instrument_failure"
|
||||
assert "gVisor denied namespace" in record["exception"]
|
||||
assert record["oracles"] == {}
|
||||
|
||||
|
||||
def test_packaged_pairs_and_three_demos(tmp_path):
|
||||
for family in probe.FAMILIES:
|
||||
sides, demos = probe.materialise(tmp_path / family, family)
|
||||
assert len(demos) == 3
|
||||
assert all((s.checkout / "app.py").stat().st_size > 0 for s in sides)
|
||||
assert (sides[1].checkout / "fixed").exists()
|
||||
assert json.loads((demos["real"] / "request.json").read_text())[0] == family
|
||||
|
||||
|
||||
def test_batch_workload_and_all_lanes(tmp_path, monkeypatch):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from benchmarks.ax import search_probe as adapter
|
||||
from benchmarks.ax.batch import manifest
|
||||
|
||||
image = "registry/search@sha256:" + "a" * 64
|
||||
item = adapter.probe_item(image)
|
||||
workload = adapter.probe_workload(image)
|
||||
doc = manifest(
|
||||
"ousast-engine-search-probe-123456789012", workload, item, {"RESULT_URL": "https://objects.example.org/probe"}, "default"
|
||||
)
|
||||
assert doc["spec"]["command"] == ["python3", "-m", "openultrasast.search.probe"]
|
||||
assert item.inputs == {}
|
||||
called = []
|
||||
|
||||
def lane(args, workload):
|
||||
called.append(args.lane)
|
||||
return lambda *args: {"status": "ok"}
|
||||
|
||||
monkeypatch.setattr(adapter, "AXLane", lane)
|
||||
monkeypatch.setattr(adapter, "DockerLane", lane)
|
||||
for name in ("ax", "kind", "docker"):
|
||||
args = SimpleNamespace(image=image, lane=name, deadline=900, out=tmp_path / name)
|
||||
assert not adapter.run_probe(args)
|
||||
assert called == ["ax", "kind", "docker"]
|
||||
|
||||
|
||||
def test_probe_runs_all_fifteen_verifications(tmp_path, monkeypatch):
|
||||
from openultrasast.search.verify import SideRecord, VerificationRecord
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(probe._sandbox, "isolation_check", lambda: None)
|
||||
monkeypatch.setattr(probe, "image_facts", lambda: {"python": {"version": "test"}})
|
||||
monkeypatch.setattr(probe, "BrowserExecutor", lambda: object())
|
||||
|
||||
def verify(a, b, demo, family, **kwargs):
|
||||
calls.append((family, demo.name))
|
||||
return VerificationRecord(
|
||||
"demonstrated" if demo.name == "real" else "inconclusive", (SideRecord("observed", (), 1, "", 0.2, 0.3, 0.5),) * 2, 2, ""
|
||||
)
|
||||
|
||||
monkeypatch.setattr(probe, "verify", verify)
|
||||
record = probe.probe(tmp_path)
|
||||
assert record["status"] == "ok"
|
||||
assert len(calls) == 15
|
||||
assert record["oracles"]["sql"]["real"]["sides"][0]["ready_seconds"] == 0.3
|
||||
assert record["scratch_disk_bytes"] > 0
|
||||
|
||||
|
||||
def test_result_put_same_record_without_credentials(monkeypatch, capsys):
|
||||
from io import BytesIO
|
||||
|
||||
monkeypatch.setenv("RESULT_URL", "https://store.example/probe?signature=secret")
|
||||
monkeypatch.setattr(probe, "probe", lambda root: {"status": "ok"})
|
||||
seen = []
|
||||
|
||||
class Response(BytesIO):
|
||||
status = 200
|
||||
|
||||
def upload(request, **kwargs):
|
||||
seen.append(request)
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr(probe.urllib.request, "urlopen", upload)
|
||||
assert probe.main([]) == 0
|
||||
assert seen[0].data == capsys.readouterr().out.encode()
|
||||
assert seen[0].method == "PUT"
|
||||
assert not seen[0].has_header("Authorization")
|
||||
|
||||
|
||||
def test_oracle_namespace_failure_is_instrument_failure(tmp_path, monkeypatch):
|
||||
from openultrasast.search.verify import SideRecord, VerificationRecord
|
||||
|
||||
monkeypatch.setattr(probe._sandbox, "isolation_check", lambda: None)
|
||||
monkeypatch.setattr(probe, "image_facts", lambda: {})
|
||||
monkeypatch.setattr(
|
||||
probe,
|
||||
"verify",
|
||||
lambda *a, **kw: VerificationRecord(
|
||||
"inconclusive", (SideRecord("could_not_run", (), 0, "verification unavailable or refused: net namespace denied"),), 0, ""
|
||||
),
|
||||
)
|
||||
record = probe.probe(tmp_path)
|
||||
assert record["status"] == "instrument_failure"
|
||||
assert "net namespace denied" in record["exception"]
|
||||
assert record["isolation_check"]["status"] == "ok"
|
||||
@@ -10,7 +10,7 @@ import pytest
|
||||
|
||||
from openultrasast.search.verify import Side, verify
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures/search/app.py"
|
||||
FIXTURE = Path(__file__).parents[1] / "src/openultrasast/search/probe_apps/app.py.txt"
|
||||
PAYLOADS = {
|
||||
"path": "../canary",
|
||||
"sql": "' OR public=0 --",
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.plane.memory import FileStore
|
||||
from openultrasast.search.board import Board
|
||||
from openultrasast.search.budget import SearchBudget
|
||||
from openultrasast.search.coordinator import Coordinator, SearchTask
|
||||
from openultrasast.search.probe import materialise
|
||||
from openultrasast.search.verify import verify
|
||||
from openultrasast.search.worker import StubModel, Worker
|
||||
|
||||
|
||||
def reply(value):
|
||||
return {"content": json.dumps(value)}
|
||||
|
||||
|
||||
def tool(name, **arguments):
|
||||
return {"tool_calls": [{"id": "call-1", "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}}]}
|
||||
|
||||
|
||||
def task(step="reason"):
|
||||
return SearchTask("task-1", "search-1", step, "facts: []\nhints: []", {}, asdict(SearchBudget()))
|
||||
|
||||
|
||||
def test_reason_no_tools_and_bounded(tmp_path):
|
||||
model = StubModel([reply({"intents": [{"description": "inspect", "from_facts": []}]})])
|
||||
result = Worker(model, repo=tmp_path, demo=tmp_path / "demo")(task())
|
||||
assert result["intents"][0]["description"] == "inspect"
|
||||
assert model.calls[0]["tools"] == []
|
||||
assert result["cost_usd"] == 0
|
||||
model = StubModel([reply({"intents": [{"description": "x"}] * 4})])
|
||||
assert Worker(model, repo=tmp_path, demo=tmp_path / "demo")(task())["status"] == "execution_failure"
|
||||
|
||||
|
||||
def test_tools_bounds_and_path_escape(tmp_path):
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "many").write_text("needle\n" * 1000)
|
||||
(repo / "link").symlink_to("/etc/passwd")
|
||||
worker = Worker(StubModel([]), repo=repo, demo=tmp_path / "demo")
|
||||
assert len(worker.tool("read_file", {"path": "many", "lines": 900})["text"].splitlines()) == 400
|
||||
assert len(worker.tool("grep", {"pattern": "needle"})["matches"]) == 200
|
||||
for path in ("../escape", "/etc/passwd", "link"):
|
||||
with pytest.raises(ValueError):
|
||||
worker.tool("read_file", {"path": path})
|
||||
with pytest.raises(ValueError):
|
||||
worker.tool("write_demo", {"path": "../escape", "content": "bad"})
|
||||
|
||||
|
||||
def test_run_scrubs_model_key_in_real_sandbox(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "must-not-reach-repo")
|
||||
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo")
|
||||
result = worker.tool(
|
||||
"run",
|
||||
{
|
||||
"command": ["/usr/bin/python3", "-I", "-c", 'import os; print("OPENROUTER_API_KEY" in os.environ); print(os.getuid())'],
|
||||
"timeout_seconds": 2,
|
||||
},
|
||||
)
|
||||
assert result["exit_code"] == 0, result
|
||||
assert result["stdout"].splitlines()[0] == "False"
|
||||
assert "must-not-reach-repo" not in json.dumps(result)
|
||||
|
||||
|
||||
def test_coordinator_stub_smoke_real_verify(tmp_path):
|
||||
sides, _ = materialise(tmp_path / "pair", "path")
|
||||
demo = tmp_path / "demo"
|
||||
model = StubModel(
|
||||
[
|
||||
reply({"intents": [{"description": "prove path traversal", "from_facts": []}]}),
|
||||
tool("read_file", path="app.py"),
|
||||
tool("write_demo", path="build.sh", content="test -s /workspace/app.py\n"),
|
||||
tool("write_demo", path="request.json", content='["path", "../canary"]'),
|
||||
tool("finish", fact={"text": "Traversal candidate", "evidence_refs": ["tool:1"], "demo": "demo/"}),
|
||||
]
|
||||
)
|
||||
worker = Worker(model, repo=sides[0].checkout, demo=demo)
|
||||
outcomes = []
|
||||
|
||||
def executor(job):
|
||||
if job.step != "verify":
|
||||
return worker(job)
|
||||
result = verify(*sides, demo, "path")
|
||||
outcomes.append(result.outcome)
|
||||
return {"status": "ok", "outcome": result.outcome, "evidence_refs": ["verify:local"], "cost_usd": 0}
|
||||
|
||||
board = Board(FileStore(tmp_path / "memory"), "smoke", coordinator="host")
|
||||
result = Coordinator(board, executor).run()
|
||||
assert outcomes == ["demonstrated"]
|
||||
assert result["end_reason"] == "goal_met"
|
||||
assert result["facts"][-1]["step"] == "verify"
|
||||
|
||||
|
||||
def test_reservation_precedes_each_call_and_refuses_over_budget(tmp_path):
|
||||
from openultrasast.model.endpoint import Prices
|
||||
from openultrasast.plane.budget import SpendBudget
|
||||
|
||||
class MeteredStub(StubModel):
|
||||
def complete_chat_raw(self, **kwargs):
|
||||
assert meter.reserved > 0
|
||||
return super().complete_chat_raw(**kwargs)
|
||||
|
||||
client = MeteredStub([reply({}), reply({})])
|
||||
worker = Worker(client, repo=tmp_path, demo=tmp_path / "demo", prices=Prices(1, 1, 1))
|
||||
meter = SpendBudget(0.1)
|
||||
assert worker.reason("facts: []", meter) == {}
|
||||
assert meter.reserved == 0
|
||||
job = task()
|
||||
job.limits["max_call_usd"] = 0.000001
|
||||
result = worker(job)
|
||||
assert result["failure"] == "BudgetExhausted"
|
||||
assert len(client.calls) == 1
|
||||
|
||||
|
||||
def test_failed_model_call_charged_and_secrets_not_reported(tmp_path):
|
||||
from openultrasast.model.endpoint import Prices
|
||||
|
||||
class Broken(StubModel):
|
||||
def complete_chat_raw(self, **kwargs):
|
||||
raise RuntimeError("Authorization: super-private-key")
|
||||
|
||||
result = Worker(Broken([]), repo=tmp_path, demo=tmp_path / "demo", prices=Prices(1, 1, 1))(task())
|
||||
assert result["cost_usd"] > 0
|
||||
assert result["status"] == "execution_failure"
|
||||
assert "super-private-key" not in json.dumps(result)
|
||||
|
||||
|
||||
def test_run_timeout_truncation_and_disk_bound(tmp_path):
|
||||
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo")
|
||||
result = worker.tool("run", {"command": ["/bin/sleep", "3"], "timeout_seconds": 1})
|
||||
assert result["timed_out"]
|
||||
result = worker.tool("run", {"command": ["/usr/bin/python3", "-I", "-c", 'print("x" * 20000)']})
|
||||
assert result["truncated"] and len(result["stdout"]) == 16384
|
||||
worker.limits["disk_bytes"] = 4096
|
||||
result = worker.tool("run", {"command": ["/usr/bin/python3", "-I", "-c", 'open("/scratch/full", "wb").write(b"x" * 16384)']})
|
||||
assert result["exit_code"] != 0
|
||||
assert "No space left" in result["stderr"]
|
||||
|
||||
|
||||
def test_finish_requires_actual_evidence_and_single_fact(tmp_path):
|
||||
for response in (
|
||||
tool("finish", fact={"text": "fake", "evidence_refs": ["missing"]}),
|
||||
tool("finish", fact={"text": "x", "evidence_refs": []}, failure="both"),
|
||||
):
|
||||
result = Worker(StubModel([response]), repo=tmp_path, demo=tmp_path / "demo")(task("explore"))
|
||||
assert result["status"] == "execution_failure"
|
||||
assert "fact" not in result
|
||||
|
||||
|
||||
def test_reason_completion_cites_board_facts(tmp_path):
|
||||
good = {"complete": {"from": ["f1"], "description": "Candidate ready"}}
|
||||
worker = Worker(StubModel([reply(good)]), repo=tmp_path, demo=tmp_path / "demo")
|
||||
job = task()
|
||||
from dataclasses import replace
|
||||
|
||||
assert worker(replace(job, snapshot="facts: [{id: f1}]"))["complete"] == good["complete"]
|
||||
Reference in New Issue
Block a user