mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5.5
85f4f0b3d4
Demos are data the verifier executes (fixed build recipes, runtime+path start, HTTP/CLI steps with captures; the canary is never referable). The agent loop with the key never runs repository code: executor tasks without a key exchange commands and results through presigned objects. AX capacity errors are back-pressure with at most 2 tasks in flight; caches and builds under /workspace with a 2 GiB guard; verifiers take one presigned archive per side. Launcher fix: private /dev in bwrap. 190 tests on the host; probe 5/5 demonstrated. Co-Authored-By: Claude Opus 5.5 <[email protected]>
232 lines
9.7 KiB
Python
232 lines
9.7 KiB
Python
import json
|
|
from dataclasses import asdict
|
|
|
|
import pytest
|
|
|
|
from openultrasast.plane.memory import FileStore
|
|
from openultrasast.search import _sandbox
|
|
from openultrasast.search.board import Board
|
|
from openultrasast.search.budget import SearchBudget
|
|
from openultrasast.search.coordinator import Coordinator, SearchTask
|
|
from openultrasast.search.executor import InProcessExecutor
|
|
from openultrasast.search.probe import materialise
|
|
from openultrasast.search.verify import verify
|
|
from openultrasast.search.worker import StubModel, Worker
|
|
|
|
|
|
def reply(value):
|
|
return {"content": json.dumps(value)}
|
|
|
|
|
|
def tool(name, **arguments):
|
|
return {"tool_calls": [{"id": "call-1", "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}}]}
|
|
|
|
|
|
def task(step="reason"):
|
|
return SearchTask("task-1", "search-1", step, "facts: []\nhints: []", {}, asdict(SearchBudget()))
|
|
|
|
|
|
def test_reason_no_tools_and_bounded(tmp_path):
|
|
model = StubModel([reply({"intents": [{"description": "inspect", "from_facts": []}]})])
|
|
result = Worker(model, repo=tmp_path, demo=tmp_path / "demo")(task())
|
|
assert result["intents"][0]["description"] == "inspect"
|
|
assert model.calls[0]["tools"] == []
|
|
assert result["isolation_mode"] == "brain-only"
|
|
assert result["cost_usd"] == 0
|
|
model = StubModel([reply({"intents": [{"description": "x"}] * 4})])
|
|
assert Worker(model, repo=tmp_path, demo=tmp_path / "demo")(task())["status"] == "execution_failure"
|
|
|
|
|
|
def test_tools_bounds_and_path_escape(tmp_path):
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "many").write_text("needle\n" * 1000)
|
|
(repo / "link").symlink_to("/etc/passwd")
|
|
worker = Worker(StubModel([]), repo=repo, demo=tmp_path / "demo", executor=InProcessExecutor(repo))
|
|
assert len(worker.tool("read_file", {"path": "many", "lines": 900})["text"].splitlines()) == 400
|
|
assert len(worker.tool("grep", {"pattern": "needle"})["matches"]) == 200
|
|
for path in ("../escape", "/etc/passwd", "link"):
|
|
with pytest.raises(ValueError):
|
|
worker.tool("read_file", {"path": path})
|
|
with pytest.raises(ValueError):
|
|
worker.tool("write_demo", {"path": "../escape", "content": "bad"})
|
|
|
|
|
|
def test_run_scrubs_model_key_in_real_sandbox(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "must-not-reach-repo")
|
|
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path))
|
|
result = worker.tool(
|
|
"run",
|
|
{
|
|
"command": ["/usr/bin/python3", "-I", "-c", 'import os; print("OPENROUTER_API_KEY" in os.environ); print(os.getuid())'],
|
|
"timeout_seconds": 2,
|
|
},
|
|
)
|
|
assert result["isolation_mode"] == _sandbox.isolation_mode()
|
|
assert result["exit_code"] == 0, result
|
|
assert result["stdout"].splitlines()[0] == "False"
|
|
assert "must-not-reach-repo" not in json.dumps(result)
|
|
|
|
|
|
def test_coordinator_stub_smoke_real_verify(tmp_path):
|
|
sides, probe_demos = materialise(tmp_path / "pair", "path")
|
|
demo = tmp_path / "demo"
|
|
model = StubModel(
|
|
[
|
|
reply({"intents": [{"description": "prove path traversal", "from_facts": []}]}),
|
|
tool("read_file", path="app.py"),
|
|
tool("write_demo", schema=json.loads((probe_demos["real"] / "demo.json").read_text())),
|
|
tool("finish", fact={"text": "Traversal candidate", "evidence_refs": ["tool:1"], "demo": "demo/"}),
|
|
]
|
|
)
|
|
worker = Worker(model, repo=sides[0].checkout, demo=demo, executor=InProcessExecutor(sides[0].checkout))
|
|
outcomes = []
|
|
|
|
def executor(job):
|
|
if job.step != "verify":
|
|
return worker(job)
|
|
result = verify(*sides, demo, "path")
|
|
outcomes.append(result.outcome)
|
|
return {"status": "ok", "outcome": result.outcome, "evidence_refs": ["verify:local"], "cost_usd": 0}
|
|
|
|
board = Board(FileStore(tmp_path / "memory"), "smoke", coordinator="host")
|
|
result = Coordinator(board, executor).run()
|
|
assert outcomes == ["demonstrated"]
|
|
assert result["end_reason"] == "goal_met"
|
|
assert result["facts"][-1]["step"] == "verify"
|
|
|
|
|
|
def test_reservation_precedes_each_call_and_refuses_over_budget(tmp_path):
|
|
from openultrasast.model.endpoint import Prices
|
|
from openultrasast.plane.budget import SpendBudget
|
|
|
|
class MeteredStub(StubModel):
|
|
def complete_chat_raw(self, **kwargs):
|
|
assert meter.reserved > 0
|
|
return super().complete_chat_raw(**kwargs)
|
|
|
|
client = MeteredStub([reply({}), reply({})])
|
|
worker = Worker(client, repo=tmp_path, demo=tmp_path / "demo", prices=Prices(1, 1, 1))
|
|
meter = SpendBudget(0.1)
|
|
assert worker.reason("facts: []", meter) == {}
|
|
assert meter.reserved == 0
|
|
job = task()
|
|
job.limits["max_call_usd"] = 0.000001
|
|
result = worker(job)
|
|
assert result["failure"] == "BudgetExhausted"
|
|
assert len(client.calls) == 1
|
|
|
|
|
|
def test_failed_model_call_charged_and_secrets_not_reported(tmp_path):
|
|
from openultrasast.model.endpoint import Prices
|
|
|
|
class Broken(StubModel):
|
|
def complete_chat_raw(self, **kwargs):
|
|
raise RuntimeError("Authorization: super-private-key")
|
|
|
|
result = Worker(Broken([]), repo=tmp_path, demo=tmp_path / "demo", prices=Prices(1, 1, 1))(task())
|
|
assert result["cost_usd"] > 0
|
|
assert result["status"] == "execution_failure"
|
|
assert "super-private-key" not in json.dumps(result)
|
|
|
|
|
|
def test_run_timeout_truncation_and_disk_bound(tmp_path):
|
|
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path))
|
|
result = worker.tool("run", {"command": ["/bin/sleep", "3"], "timeout_seconds": 1})
|
|
assert result["timed_out"]
|
|
result = worker.tool("run", {"command": ["/usr/bin/python3", "-I", "-c", 'print("x" * 20000)']})
|
|
assert result["truncated"] and len(result["stdout"]) == 16384
|
|
worker.limits["disk_bytes"] = 4096
|
|
result = worker.tool("run", {"command": ["/usr/bin/python3", "-I", "-c", 'open("/scratch/full", "wb").write(b"x" * 16384)']})
|
|
assert result["exit_code"] != 0
|
|
assert "No space left" in result["stderr"]
|
|
|
|
|
|
def test_finish_requires_actual_evidence_and_single_fact(tmp_path):
|
|
for response in (
|
|
tool("finish", fact={"text": "fake", "evidence_refs": ["missing"]}),
|
|
tool("finish", fact={"text": "x", "evidence_refs": []}, failure="both"),
|
|
):
|
|
result = Worker(StubModel([response]), repo=tmp_path, demo=tmp_path / "demo")(task("explore"))
|
|
assert result["status"] == "execution_failure"
|
|
assert "fact" not in result
|
|
|
|
|
|
def test_reason_completion_cites_board_facts(tmp_path):
|
|
good = {"complete": {"from": ["f1"], "description": "Candidate ready"}}
|
|
worker = Worker(StubModel([reply(good)]), repo=tmp_path, demo=tmp_path / "demo")
|
|
job = task()
|
|
from dataclasses import replace
|
|
|
|
assert worker(replace(job, snapshot="facts: [{id: f1}]"))["complete"] == good["complete"]
|
|
|
|
|
|
def test_brain_routes_every_repository_tool_without_local_access(tmp_path, monkeypatch):
|
|
class RecordingExecutor:
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
def submit(self, name, args, **kwargs):
|
|
self.calls.append((name, args, kwargs))
|
|
return {"executor": name}
|
|
|
|
executor = RecordingExecutor()
|
|
worker = Worker(StubModel([]), repo=tmp_path / "absent", demo=tmp_path / "demo", executor=executor)
|
|
import subprocess
|
|
|
|
def forbidden(*args, **kwargs):
|
|
raise AssertionError("brain touched repository")
|
|
|
|
monkeypatch.setattr(subprocess, "Popen", forbidden)
|
|
monkeypatch.setattr(type(tmp_path), "open", forbidden)
|
|
for name, args in (
|
|
("list_files", {}),
|
|
("read_file", {"path": "app.py"}),
|
|
("grep", {"pattern": "needle"}),
|
|
("run", {"command": ["python3", "app.py"]}),
|
|
):
|
|
assert worker.tool(name, args) == {"executor": name}
|
|
assert [c[0] for c in executor.calls] == ["list_files", "read_file", "grep", "run"]
|
|
|
|
|
|
def test_brain_refuses_repository_tools_without_executor(tmp_path):
|
|
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo")
|
|
with pytest.raises(ValueError, match="executor"):
|
|
worker.tool("list_files", {})
|
|
|
|
|
|
def test_write_demo_accepts_only_declarative_json(tmp_path):
|
|
worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo")
|
|
schema = {
|
|
"build": {"recipe": "none", "arguments": []},
|
|
"start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
|
|
"steps": [{"type": "cli", "arguments": ["hello"]}],
|
|
}
|
|
assert worker.tool("write_demo", {"schema": schema})["path"] == "demo/demo.json"
|
|
assert json.loads((tmp_path / "demo/demo.json").read_text()) == schema
|
|
with pytest.raises(ValueError):
|
|
worker.tool("write_demo", {"path": "build.sh", "content": "touch /tmp/pwn"})
|
|
|
|
|
|
def test_inprocess_executor_observes_all_brain_repository_commands(tmp_path):
|
|
class RecordingExecutor(InProcessExecutor):
|
|
def __init__(self, repo):
|
|
super().__init__(repo)
|
|
self.commands = []
|
|
|
|
def execute(self, command):
|
|
self.commands.append(command)
|
|
return super().execute(command)
|
|
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "sample.txt").write_text("needle")
|
|
executor = RecordingExecutor(repo)
|
|
worker = Worker(StubModel([]), repo=repo, demo=tmp_path / "demo", executor=executor)
|
|
assert worker.tool("list_files", {})["files"] == ["sample.txt"]
|
|
assert worker.tool("read_file", {"path": "sample.txt"})["input_bytes"] == 6
|
|
assert worker.tool("grep", {"pattern": "needle"})["matches"][0]["line"] == 1
|
|
assert worker.tool("run", {"command": ["/bin/cat", "/workspace/sample.txt"]})["stdout"] == "needle"
|
|
assert [c.name for c in executor.commands] == ["list_files", "read_file", "grep", "run"]
|
|
assert [c.seq for c in executor.commands] == [1, 2, 3, 4]
|