mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Fable 5.1
f5a904b7f2
Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts). Deleted: src/openultrasast/regress/ and the deep-mode REGRESS branch (deep now records a regress_retired degradation and equals standard; proof is the search-with-proof arm's job), the sandbox docker runner and probe that only regress used, --fail-on worth-fixing, RegressConfig, the two ledger helpers only regress called; skills.py and index.py with the hunter pool's skill routing; model/audit.py; cpg/queries/vocabulary.sc with benchmarks/push/ontology_gap.py. Docs and the scan skill describe the retirement. Net: 3,895 lines deleted, 86 added. Targeted tests: 180 passed, 28 skipped; 3,331 tests collect. Co-Authored-By: Claude Fable 5.1 <[email protected]>
245 lines
8.9 KiB
Python
245 lines
8.9 KiB
Python
"""Offline replay through real preparation, multipart export and owned oracles."""
|
|
|
|
import json
|
|
import shutil
|
|
import sys
|
|
import time
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from benchmarks.ax import batch
|
|
from benchmarks.search import demo_replay, pilot_run
|
|
from openultrasast.search import executor as executor_module
|
|
from openultrasast.search.executor import InProcessExecutor
|
|
from openultrasast.search.probe import materialise
|
|
from openultrasast.search.verify import Side, verify_side_task
|
|
from openultrasast.search.verify_task import extract_checkout
|
|
|
|
IMAGE = "image@sha256:" + "a" * 64
|
|
|
|
|
|
def harness(tmp_path, monkeypatch, *, failing=False):
|
|
sides, demos = materialise(tmp_path / "probe", "path")
|
|
events, objects = [], {}
|
|
|
|
class Transport:
|
|
def put(self, url, data, timeout):
|
|
objects[url] = data if isinstance(data, bytes) else data.read()
|
|
|
|
monkeypatch.setattr(executor_module, "URLTransport", Transport)
|
|
|
|
class Executor(InProcessExecutor):
|
|
def _task_run(self, argv, timeout, limits, **kwargs):
|
|
if argv[:2] == ["sh", "-ec"]:
|
|
assert argv[2] == pilot_run.CLONE
|
|
source = sides[argv[-1] == "b" * 40].checkout
|
|
shutil.copytree(source, self.repo, dirs_exist_ok=True)
|
|
if failing:
|
|
(self.repo / "build.gradle").write_text("// offline failing fixture\n")
|
|
(self.repo / "gradlew").write_text("echo replay build broke >&2\nexit 7\n")
|
|
return dict(exit_code=0, stdout="cloned", stderr="")
|
|
return super()._task_run(argv, timeout, limits, **kwargs)
|
|
|
|
class Task:
|
|
prepare = batch.SearchExecutorTask.prepare
|
|
|
|
def __init__(self, lane, archive, deadline):
|
|
self.lane = SimpleNamespace(store=SimpleNamespace(_get=lambda key: (objects[key], {}) if key in objects else None))
|
|
self.keys = ["unused0", "unused1", "unused2", "manifest", "part"]
|
|
self.end = time.monotonic() + deadline
|
|
|
|
def __enter__(self):
|
|
events.append("executor")
|
|
repo = tmp_path / ("exec-" + str(len(events)))
|
|
repo.mkdir()
|
|
self.client = Executor(repo, task_boundary=True)
|
|
self.client.prepared_put_url = "manifest"
|
|
self.client.prepared_part_urls = ["part"]
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
events.append("deleted")
|
|
|
|
monkeypatch.setattr(pilot_run, "SearchExecutorTask", Task)
|
|
ids = []
|
|
|
|
def lane(item, output, attempt):
|
|
assert events[-1] in ("deleted", "verify")
|
|
events.append("verify")
|
|
assert item.id not in ids
|
|
ids.append(item.id)
|
|
bundle = output / "bundle"
|
|
extract_checkout(item.inputs["VERIFY_INPUT_URL"], bundle)
|
|
spec = json.loads((bundle / "spec.json").read_text())
|
|
assert spec["demo"]["build"] == {"recipe": "none", "arguments": []}
|
|
demo = output / "demo.json"
|
|
demo.write_text(json.dumps(spec["demo"]))
|
|
return asdict(
|
|
verify_side_task(Side(bundle / "checkout", products=bundle / "products"), demo, spec["family"], spec["timeout_seconds"])
|
|
)
|
|
|
|
schema_path = demos["real"] / "demo.json"
|
|
if failing:
|
|
schema = json.loads(schema_path.read_text())
|
|
schema["build"] = {"recipe": "gradle", "arguments": ["build.gradle"]}
|
|
schema_path.write_text(json.dumps(schema))
|
|
args = SimpleNamespace(
|
|
repo_url="https://github.com/private-owner/private-project",
|
|
commit="a" * 40,
|
|
fixed_commit="b" * 40,
|
|
demo=schema_path,
|
|
family="path",
|
|
lane="ax",
|
|
image=IMAGE,
|
|
both=True,
|
|
)
|
|
root = tmp_path / "replay"
|
|
root.mkdir()
|
|
return args, root, lane, events
|
|
|
|
|
|
def test_both_cli_demonstrates_probe_pair(tmp_path, monkeypatch):
|
|
args, root, lane, events = harness(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(demo_replay, "AXLane", lambda *a: lane)
|
|
out = tmp_path / "out"
|
|
assert (
|
|
demo_replay.main(
|
|
[
|
|
"--repo-url",
|
|
args.repo_url,
|
|
"--commit",
|
|
args.commit,
|
|
"--fixed-commit",
|
|
args.fixed_commit,
|
|
"--both",
|
|
"--demo",
|
|
str(args.demo),
|
|
"--family",
|
|
"path",
|
|
"--lane",
|
|
"ax",
|
|
"--image",
|
|
IMAGE,
|
|
"--out",
|
|
str(out),
|
|
]
|
|
)
|
|
== 0
|
|
)
|
|
record = json.loads((out / "record.json").read_text())
|
|
assert record["outcome"] == "demonstrated", record
|
|
assert events == ["executor", "deleted", "verify", "verify", "verify"] * 2
|
|
assert [[r["observed"] for r in s["runs"]] for s in record["sides"]] == [[True] * 3, [False] * 3]
|
|
assert all(r["evidence"] for s in record["sides"] for r in s["runs"])
|
|
assert record["input_bytes"] > 0
|
|
assert record["model_calls"] == record["spend_usd"] == 0
|
|
for prep in record["preparations"]:
|
|
assert prep["archive_bytes"] > 0 and prep["archive_parts"] == 1
|
|
assert prep["build_exit_code"] == 0
|
|
assert {"acquisition", "build", "bundle", "upload", "prepare_roundtrip"} <= prep["wall_seconds_by_phase"].keys()
|
|
assert args.repo_url not in json.dumps(record)
|
|
assert args.commit not in json.dumps(record)
|
|
assert args.fixed_commit not in json.dumps(record)
|
|
|
|
|
|
def test_single_build_failure_retains_stderr(tmp_path, monkeypatch):
|
|
args, root, lane, events = harness(tmp_path, monkeypatch, failing=True)
|
|
args.both, args.fixed_commit = False, None
|
|
record = demo_replay.run_replay(args, root, lane=lane)
|
|
assert record["outcome"] == "could_not_build", record
|
|
assert record["sides"][0]["outcome"] == "could_not_build"
|
|
assert record["preparations"][0]["build_exit_code"] != 0
|
|
assert "replay build broke" in record["preparations"][0]["build_stderr_tail"]
|
|
assert events == ["executor", "deleted"]
|
|
|
|
|
|
@pytest.mark.parametrize("fixed", [False, True])
|
|
def test_single_observation_is_not_a_differential(tmp_path, monkeypatch, fixed):
|
|
args, root, lane, events = harness(tmp_path, monkeypatch)
|
|
args.both = False
|
|
args.commit = args.fixed_commit if fixed else args.commit
|
|
args.fixed_commit = None
|
|
record = demo_replay.run_replay(args, root, lane=lane)
|
|
assert record["outcome"] == "observed", record
|
|
assert [r["observed"] for r in record["sides"][0]["runs"]] == [not fixed] * 3
|
|
|
|
|
|
def test_no_oracle_does_not_acquire(tmp_path, monkeypatch):
|
|
args, root, lane, events = harness(tmp_path, monkeypatch)
|
|
args.family = "access_control"
|
|
record = demo_replay.run_replay(args, root, lane=lane)
|
|
assert record["outcome"] == "no_oracle"
|
|
assert not events
|
|
|
|
|
|
def test_both_identical_revisions_are_inconclusive(tmp_path, monkeypatch):
|
|
args, root, lane, events = harness(tmp_path, monkeypatch)
|
|
args.fixed_commit = args.commit
|
|
record = demo_replay.run_replay(args, root, lane=lane)
|
|
assert record["outcome"] == "inconclusive"
|
|
assert record["verify_tasks"] == 6 and record["tasks_submitted"] == 8
|
|
|
|
|
|
def test_lane_failure_is_could_not_run(tmp_path, monkeypatch):
|
|
args, root, lane, events = harness(tmp_path, monkeypatch)
|
|
args.both, args.fixed_commit = False, None
|
|
|
|
def broken(*args):
|
|
raise ValueError("verifier unavailable")
|
|
|
|
record = demo_replay.run_replay(args, root, lane=broken)
|
|
assert record["outcome"] == "could_not_run"
|
|
assert "verifier unavailable" in record["reason"]
|
|
|
|
|
|
def test_docker_selection_uses_same_preparation_and_fresh_runs(tmp_path, monkeypatch):
|
|
args, root, fake_lane, events = harness(tmp_path, monkeypatch)
|
|
args.lane = "docker"
|
|
|
|
class Docker(batch.DockerLane):
|
|
def __init__(self, *args):
|
|
pass
|
|
|
|
def _prepare_image(self):
|
|
events.append("docker-image")
|
|
|
|
def __call__(self, *args):
|
|
return fake_lane(*args)
|
|
|
|
monkeypatch.setattr(demo_replay, "DockerLane", Docker)
|
|
record = demo_replay.run_replay(args, root)
|
|
assert record["outcome"] == "demonstrated"
|
|
assert events[0] == "docker-image"
|
|
assert record["executor_tasks"] == 2 and record["verify_tasks"] == 6
|
|
|
|
|
|
def test_docker_verifier_command_has_no_engine_arguments():
|
|
import subprocess
|
|
|
|
commands = []
|
|
lane = batch.DockerLane.__new__(batch.DockerLane)
|
|
lane._diagnostics = []
|
|
|
|
def runner(command, **kwargs):
|
|
commands.append(command)
|
|
return subprocess.CompletedProcess(command, 0, "container", "")
|
|
|
|
lane.runner = runner
|
|
lane._docker_command(
|
|
"apply",
|
|
manifest={
|
|
"metadata": {"name": "verifier"},
|
|
"spec": {
|
|
"command": ["python3", "-m", "openultrasast.search.verify_task"],
|
|
"env": [],
|
|
"image": IMAGE,
|
|
},
|
|
},
|
|
)
|
|
assert commands[0][-3:] == [IMAGE, "-m", "openultrasast.search.verify_task"]
|
|
assert "--heap-profile" not in commands[0]
|