search-with-proof: no-model demo replay (executor + verifier on a stored demo)

Replays a stored demo through the executor task and the verifier lane for one side or both, with per-phase timings, so verifier changes are checked on the exact image before any paid search. Co-Authored-By: Claude Fable 5.1 <[email protected]>
2026-10-16 03:49:26 +00:00 · 2026-10-06 14:50:14 +02:00
parent 630603ee28
commit eb05ed18da
5 changed files with 513 additions and 2 deletions
@@ -0,0 +1,251 @@
+"""Offline replay through real preparation, multipart export and owned oracles."""
+
+import json
+import shutil
+import sys
+import time
+from dataclasses import asdict
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from benchmarks.ax import batch
+from benchmarks.search import demo_replay, pilot_run
+from openultrasast.search import executor as executor_module
+from openultrasast.search.executor import InProcessExecutor
+from openultrasast.search.probe import materialise
+from openultrasast.search.verify import Side, verify_side_task
+from openultrasast.search.verify_task import extract_checkout
+
+IMAGE = "image@sha256:" + "a" * 64
+
+
+def harness(tmp_path, monkeypatch, *, failing=False):
+    sides, demos = materialise(tmp_path / "probe", "path")
+    events, objects = [], {}
+
+    class Transport:
+        def put(self, url, data, timeout):
+            objects[url] = data if isinstance(data, bytes) else data.read()
+
+    monkeypatch.setattr(executor_module, "URLTransport", Transport)
+
+    class Executor(InProcessExecutor):
+        def _task_run(self, argv, timeout, limits, **kwargs):
+            if argv[:2] == ["sh", "-ec"]:
+                assert argv[2] == pilot_run.CLONE
+                source = sides[argv[-1] == "b" * 40].checkout
+                shutil.copytree(source, self.repo, dirs_exist_ok=True)
+                if failing:
+                    (self.repo / "build.gradle").write_text("// offline failing fixture\n")
+                    (self.repo / "gradlew").write_text("echo replay build broke >&2\nexit 7\n")
+                return dict(exit_code=0, stdout="cloned", stderr="")
+            return super()._task_run(argv, timeout, limits, **kwargs)
+
+    class Task:
+        prepare = batch.SearchExecutorTask.prepare
+
+        def __init__(self, lane, archive, deadline):
+            self.lane = SimpleNamespace(store=SimpleNamespace(_get=lambda key: (objects[key], {}) if key in objects else None))
+            self.keys = ["unused0", "unused1", "unused2", "manifest", "part"]
+            self.end = time.monotonic() + deadline
+
+        def __enter__(self):
+            events.append("executor")
+            repo = tmp_path / ("exec-" + str(len(events)))
+            repo.mkdir()
+            self.client = Executor(repo, task_boundary=True)
+            self.client.prepared_put_url = "manifest"
+            self.client.prepared_part_urls = ["part"]
+            return self
+
+        def __exit__(self, *args):
+            events.append("deleted")
+
+    monkeypatch.setattr(pilot_run, "SearchExecutorTask", Task)
+    ids = []
+
+    def lane(item, output, attempt):
+        assert events[-1] in ("deleted", "verify")
+        events.append("verify")
+        assert item.id not in ids
+        ids.append(item.id)
+        bundle = output / "bundle"
+        extract_checkout(item.inputs["VERIFY_INPUT_URL"], bundle)
+        spec = json.loads((bundle / "spec.json").read_text())
+        assert spec["demo"]["build"] == {"recipe": "none", "arguments": []}
+        demo = output / "demo.json"
+        demo.write_text(json.dumps(spec["demo"]))
+        return asdict(
+            verify_side_task(Side(bundle / "checkout", products=bundle / "products"), demo, spec["family"], spec["timeout_seconds"])
+        )
+
+    schema_path = demos["real"] / "demo.json"
+    if failing:
+        schema = json.loads(schema_path.read_text())
+        schema["build"] = {"recipe": "gradle", "arguments": ["build.gradle"]}
+        schema_path.write_text(json.dumps(schema))
+    args = SimpleNamespace(
+        repo_url="https://github.com/private-owner/private-project",
+        commit="a" * 40,
+        fixed_commit="b" * 40,
+        demo=schema_path,
+        family="path",
+        lane="ax",
+        image=IMAGE,
+        both=True,
+    )
+    root = tmp_path / "replay"
+    root.mkdir()
+    return args, root, lane, events
+
+
+def test_both_cli_demonstrates_probe_pair(tmp_path, monkeypatch):
+    args, root, lane, events = harness(tmp_path, monkeypatch)
+    monkeypatch.setattr(demo_replay, "AXLane", lambda *a: lane)
+    out = tmp_path / "out"
+    assert (
+        demo_replay.main(
+            [
+                "--repo-url",
+                args.repo_url,
+                "--commit",
+                args.commit,
+                "--fixed-commit",
+                args.fixed_commit,
+                "--both",
+                "--demo",
+                str(args.demo),
+                "--family",
+                "path",
+                "--lane",
+                "ax",
+                "--image",
+                IMAGE,
+                "--out",
+                str(out),
+            ]
+        )
+        == 0
+    )
+    record = json.loads((out / "record.json").read_text())
+    assert record["outcome"] == "demonstrated", record
+    assert events == ["executor", "deleted", "verify", "verify", "verify"] * 2
+    assert [[r["observed"] for r in s["runs"]] for s in record["sides"]] == [[True] * 3, [False] * 3]
+    assert all(r["evidence"] for s in record["sides"] for r in s["runs"])
+    assert record["input_bytes"] > 0
+    assert record["model_calls"] == record["spend_usd"] == 0
+    for prep in record["preparations"]:
+        assert prep["archive_bytes"] > 0 and prep["archive_parts"] == 1
+        assert prep["build_exit_code"] == 0
+        assert {"acquisition", "build", "bundle", "upload", "prepare_roundtrip"} <= prep["wall_seconds_by_phase"].keys()
+    assert args.repo_url not in json.dumps(record)
+    assert args.commit not in json.dumps(record)
+    assert args.fixed_commit not in json.dumps(record)
+
+
+def test_single_build_failure_retains_stderr(tmp_path, monkeypatch):
+    args, root, lane, events = harness(tmp_path, monkeypatch, failing=True)
+    args.both, args.fixed_commit = False, None
+    record = demo_replay.run_replay(args, root, lane=lane)
+    assert record["outcome"] == "could_not_build", record
+    assert record["sides"][0]["outcome"] == "could_not_build"
+    assert record["preparations"][0]["build_exit_code"] != 0
+    assert "replay build broke" in record["preparations"][0]["build_stderr_tail"]
+    assert events == ["executor", "deleted"]
+
+
+@pytest.mark.parametrize("fixed", [False, True])
+def test_single_observation_is_not_a_differential(tmp_path, monkeypatch, fixed):
+    args, root, lane, events = harness(tmp_path, monkeypatch)
+    args.both = False
+    args.commit = args.fixed_commit if fixed else args.commit
+    args.fixed_commit = None
+    record = demo_replay.run_replay(args, root, lane=lane)
+    assert record["outcome"] == "observed", record
+    assert [r["observed"] for r in record["sides"][0]["runs"]] == [not fixed] * 3
+
+
+def test_no_oracle_does_not_acquire(tmp_path, monkeypatch):
+    args, root, lane, events = harness(tmp_path, monkeypatch)
+    args.family = "access_control"
+    record = demo_replay.run_replay(args, root, lane=lane)
+    assert record["outcome"] == "no_oracle"
+    assert not events
+
+
+def test_both_identical_revisions_are_inconclusive(tmp_path, monkeypatch):
+    args, root, lane, events = harness(tmp_path, monkeypatch)
+    args.fixed_commit = args.commit
+    record = demo_replay.run_replay(args, root, lane=lane)
+    assert record["outcome"] == "inconclusive"
+    assert record["verify_tasks"] == 6 and record["tasks_submitted"] == 8
+
+
+def test_lane_failure_is_could_not_run(tmp_path, monkeypatch):
+    args, root, lane, events = harness(tmp_path, monkeypatch)
+    args.both, args.fixed_commit = False, None
+
+    def broken(*args):
+        raise ValueError("verifier unavailable")
+
+    record = demo_replay.run_replay(args, root, lane=broken)
+    assert record["outcome"] == "could_not_run"
+    assert "verifier unavailable" in record["reason"]
+
+
+def test_replay_registered_as_standalone():
+    from openultrasast.model.audit import audit
+
+    rows = {r.module: r for r in audit(Path("benchmarks"))}
+    assert rows["search.demo_replay"].classification == "standalone_capability"
+
+
+def test_docker_selection_uses_same_preparation_and_fresh_runs(tmp_path, monkeypatch):
+    args, root, fake_lane, events = harness(tmp_path, monkeypatch)
+    args.lane = "docker"
+
+    class Docker(batch.DockerLane):
+        def __init__(self, *args):
+            pass
+
+        def _prepare_image(self):
+            events.append("docker-image")
+
+        def __call__(self, *args):
+            return fake_lane(*args)
+
+    monkeypatch.setattr(demo_replay, "DockerLane", Docker)
+    record = demo_replay.run_replay(args, root)
+    assert record["outcome"] == "demonstrated"
+    assert events[0] == "docker-image"
+    assert record["executor_tasks"] == 2 and record["verify_tasks"] == 6
+
+
+def test_docker_verifier_command_has_no_engine_arguments():
+    import subprocess
+
+    commands = []
+    lane = batch.DockerLane.__new__(batch.DockerLane)
+    lane._diagnostics = []
+
+    def runner(command, **kwargs):
+        commands.append(command)
+        return subprocess.CompletedProcess(command, 0, "container", "")
+
+    lane.runner = runner
+    lane._docker_command(
+        "apply",
+        manifest={
+            "metadata": {"name": "verifier"},
+            "spec": {
+                "command": ["python3", "-m", "openultrasast.search.verify_task"],
+                "env": [],
+                "image": IMAGE,
+            },
+        },
+    )
+    assert commands[0][-3:] == [IMAGE, "-m", "openultrasast.search.verify_task"]
+    assert "--heap-profile" not in commands[0]