mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5.5
6d27c9bfcf
gVisor replays exhausted the hook's 30 s deadline in the snapshot step on a 1,725-file repository; cluster replays now use 30 s x a measured slowdown factor, verified on a native sample. The engine runner deletes each pin's fetched source (1.8 GB had accumulated), caps the cache at 500 MB and refuses below 1.5 GB free. Proof 4.4: two engine pins via the generic dispatcher match the VM baseline exactly. Implemented with Codex. Co-Authored-By: Claude Opus 5.5 <[email protected]>
248 lines
11 KiB
Python
248 lines
11 KiB
Python
"""Dispatch frozen unseen changes or a five-change project-history proof."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import math
|
|
import subprocess
|
|
import tarfile
|
|
import tomllib
|
|
from pathlib import Path
|
|
|
|
from benchmarks.ax.batch import AXLane, DockerLane, Item, Workload, dispatch, write_json
|
|
from benchmarks.unseen.score import coverage
|
|
|
|
|
|
def make_item(change, arm, image, *, deadline_scale=1.0, lane="ax", deadline=None):
|
|
if arm not in {"default", "long_deadline"}:
|
|
raise ValueError("unknown replay arm")
|
|
identity = hashlib.sha256(str(change["id"]).encode()).hexdigest()[:14]
|
|
if not math.isfinite(deadline_scale) or deadline_scale <= 0:
|
|
raise ValueError("deadline scale must be finite and positive")
|
|
base = deadline if deadline is not None else 30 if arm == "default" else 300
|
|
applied = round(base * (1 if lane == "docker" else deadline_scale))
|
|
if applied < 1:
|
|
raise ValueError("scaled deadline must be at least one second")
|
|
spec = {
|
|
**change,
|
|
"id": identity,
|
|
"hook_flags": ["--deadline", str(applied)],
|
|
"image": image,
|
|
"base_deadline_seconds": base,
|
|
"deadline_seconds": applied,
|
|
"deadline_scale": deadline_scale,
|
|
"applied_deadline_scale": 1.0 if lane == "docker" else deadline_scale,
|
|
}
|
|
data = json.dumps(spec, sort_keys=True).encode()
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
return Item(
|
|
"replay-" + identity,
|
|
{"SPEC_URL": data},
|
|
"result.dat",
|
|
("python3", "/app/benchmarks/unseen/replay_entry.py"),
|
|
{},
|
|
digest,
|
|
{"SPEC_URL": "spec.dat"},
|
|
)
|
|
|
|
|
|
def validate_result(data, output):
|
|
with tarfile.open(fileobj=io.BytesIO(data)) as archive:
|
|
members = archive.getmembers()
|
|
required = {"push.json", "push.txt", "engine.json", "instrument.json"}
|
|
if {m.name for m in members} != required or any(not m.isfile() for m in members):
|
|
raise ValueError("replay result requires push, engine and instrument.json")
|
|
payload = {m.name: archive.extractfile(m).read() for m in members}
|
|
instrument = json.loads(payload["instrument.json"])
|
|
push = json.loads(payload["push.json"])
|
|
record = {"instrument": instrument, "push": push}
|
|
record["status"] = coverage(record)
|
|
# Transport/scheduler needs an explicit OOM signal, not a quiet failed replay.
|
|
if "OutOfMemoryError" in json.dumps(push) or "JVM out of memory" in json.dumps(push):
|
|
record["status"] = "oom"
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
for name, body in payload.items():
|
|
(output / name).write_bytes(body)
|
|
(output / "result.dat").write_bytes(data)
|
|
write_json(output / "result.json", record)
|
|
return record
|
|
|
|
|
|
def proof_changes(root):
|
|
"""Only the operator's project history; no pool data or fetches."""
|
|
|
|
def git(*args):
|
|
return subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True, check=True).stdout.strip()
|
|
|
|
changes = []
|
|
for head in git("rev-list", "--no-merges", "--max-count=100", "HEAD", "--", "src").splitlines():
|
|
base = git("rev-parse", head + "^")
|
|
changed = git("diff", "--name-only", base, head).splitlines()
|
|
if not any(name.startswith("src/") and name.endswith(".py") for name in changed):
|
|
continue
|
|
changes.append(
|
|
{
|
|
"id": hashlib.sha256(head.encode()).hexdigest(),
|
|
"repository_url": "https://github.com/norandom/OpenUltraSAST.git",
|
|
"repo": "project-proof",
|
|
"kind": "ordinary",
|
|
"base": base,
|
|
"head": head,
|
|
}
|
|
)
|
|
if len(changes) == 5:
|
|
break
|
|
if len(changes) != 5:
|
|
raise ValueError("history lacks five source changes")
|
|
return changes
|
|
|
|
|
|
def canonical(value):
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
|
|
|
|
|
def load_slice(path, slice_id, private_path=None):
|
|
if path.suffix != ".toml":
|
|
frozen = json.loads(path.read_text())
|
|
changes = frozen["changes"]
|
|
if frozen["sha256"] != hashlib.sha256(json.dumps(changes, sort_keys=True).encode()).hexdigest():
|
|
raise ValueError("frozen changes digest mismatch")
|
|
return changes
|
|
if not path.name.startswith("pool-"):
|
|
raise ValueError("replay requires a frozen pool, never a draft")
|
|
pool = tomllib.loads(path.read_text())
|
|
freeze = json.loads(path.with_name("freeze-" + path.stem.removeprefix("pool-") + ".json").read_text())
|
|
if hashlib.sha256(canonical(pool)).hexdigest() != freeze["freeze_digest"]:
|
|
raise ValueError("frozen pool digest mismatch")
|
|
private_path = private_path or path.parent / "private" / path.name
|
|
private = {}
|
|
if private_path.exists():
|
|
private = {row["id"]: row for row in tomllib.loads(private_path.read_text()).get("repository", [])}
|
|
changes = []
|
|
for public in pool.get("repository", []):
|
|
if str(public["slice"]) != str(slice_id):
|
|
continue
|
|
row = public
|
|
if public.get("private"):
|
|
row = private.get(public["id"])
|
|
if row is None:
|
|
raise ValueError("missing private pool entry")
|
|
payload = {"url": row["url"], "commits": [[c["base"], c["head"]] for c in row["changes"]]}
|
|
if hashlib.sha256(canonical(payload)).hexdigest() != public["sha256"]:
|
|
raise ValueError("private pool digest mismatch")
|
|
for change in row["changes"]:
|
|
identity = hashlib.sha256(canonical([public["id"], change["kind"], change["base"], change["head"]])).hexdigest()
|
|
changes.append(
|
|
{
|
|
**change,
|
|
"id": identity,
|
|
"repo": public["id"],
|
|
"repository_url": row["url"],
|
|
"freeze_digest": freeze["freeze_digest"],
|
|
"slice": slice_id,
|
|
}
|
|
)
|
|
if not changes:
|
|
raise ValueError("frozen slice has no changes")
|
|
return changes
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--manifest", type=Path, required=True, help="frozen JSON: changes array plus sha256 of canonical changes JSON")
|
|
parser.add_argument("--proof-history", action="store_true", help="write a five-change own-history manifest, then exit")
|
|
parser.add_argument("--image")
|
|
parser.add_argument("--private-manifest", type=Path)
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
parser.add_argument("--arm", choices=("default", "long_deadline"), default="default")
|
|
parser.add_argument("--pool", default="p1")
|
|
parser.add_argument("--slice", default="1")
|
|
parser.add_argument("--out", type=Path)
|
|
parser.add_argument("--lane", choices=("ax", "kind", "docker", "mixed"), default="mixed")
|
|
group = parser.add_mutually_exclusive_group()
|
|
group.add_argument("--deadline-scale", type=float, default=1.0)
|
|
group.add_argument("--calibration", type=Path)
|
|
parser.add_argument("--native-sample", type=int, default=0)
|
|
parser.add_argument("--docker-image-bytes", type=int, help="unpacked image size upper bound, required for VM pull")
|
|
parser.add_argument("--ax-bin", default="ax")
|
|
parser.add_argument("--kubeconfig", type=Path)
|
|
parser.add_argument("--atespace", default="default")
|
|
args = parser.parse_args()
|
|
if args.proof_history:
|
|
changes = proof_changes(Path.cwd())
|
|
write_json(args.manifest, {"changes": changes, "sha256": hashlib.sha256(json.dumps(changes, sort_keys=True).encode()).hexdigest()})
|
|
return 0
|
|
if not args.image:
|
|
parser.error("--image digest is required")
|
|
changes = load_slice(args.manifest, args.slice, args.private_manifest)
|
|
factor = args.deadline_scale
|
|
if args.calibration:
|
|
calibration = json.loads(args.calibration.read_text())
|
|
if calibration.get("status") != "ok" or calibration.get("image") != args.image:
|
|
parser.error("calibration must be successful and match the replay image")
|
|
factor = calibration["quick_total"]["factor"]
|
|
if args.native_sample < 0 or (args.native_sample and args.lane not in {"ax", "kind"}):
|
|
parser.error("--native-sample requires an ax or kind cluster run and a positive K")
|
|
items = [make_item(change, args.arm, args.image, deadline_scale=factor, lane=args.lane) for change in changes]
|
|
if len({i.id for i in items}) != len(items):
|
|
parser.error("duplicate change identity")
|
|
args.out = args.out or Path.home() / "ousast-results/unseen" / args.pool / args.slice / args.arm
|
|
if args.dry_run:
|
|
print(json.dumps({"items": len(items), "input_digest": hashlib.sha256(canonical([i.input_digest for i in items])).hexdigest()}))
|
|
return 0
|
|
if args.out.resolve().is_relative_to(Path(__file__).resolve().parents[2]):
|
|
parser.error("raw replay output must be outside the repository")
|
|
failed = run_replays(changes, args, args.out, args.lane, deadline_scale=factor)
|
|
if args.native_sample:
|
|
failed |= run_replays(sample_changes(changes, args.native_sample), args, args.out / "native", "docker", deadline_scale=factor)
|
|
return 2 if failed else 0
|
|
|
|
|
|
def sample_changes(changes, count):
|
|
if count < 1:
|
|
raise ValueError("sample size must be positive")
|
|
if len({str(c["id"]) for c in changes}) != len(changes):
|
|
raise ValueError("duplicate change identity")
|
|
return sorted(changes, key=lambda c: hashlib.sha256(str(c["id"]).encode()).digest())[:count]
|
|
|
|
|
|
def run_replays(changes, args, out, lane, *, deadline_scale=1.0, deadline=None):
|
|
if out.resolve().is_relative_to(Path(__file__).resolve().parents[2]):
|
|
raise ValueError("raw replay output must be outside the repository")
|
|
items = [make_item(c, args.arm, args.image, deadline_scale=deadline_scale, lane=lane, deadline=deadline) for c in changes]
|
|
write_json(
|
|
out / "inputs.json",
|
|
[{"item_id": item.id, "change": change, "input_digest": item.input_digest} for item, change in zip(items, changes, strict=True)],
|
|
)
|
|
maximum = max(
|
|
max(json.loads(i.inputs["SPEC_URL"])["deadline_seconds"], json.loads(i.inputs["SPEC_URL"])["base_deadline_seconds"]) for i in items
|
|
)
|
|
workload = Workload(args.image, frozenset({"SPEC_URL", "RESULT_URL"}), maximum + 600, validate_result)
|
|
lanes = ("ax", "ax", "docker") if lane == "mixed" else (lane,) * (2 if lane in {"ax", "kind"} else 1)
|
|
original = {item.id: change for item, change in zip(items, changes, strict=True)}
|
|
executors = {}
|
|
for name in set(lanes):
|
|
transport = (DockerLane if name == "docker" else AXLane)(args, workload)
|
|
|
|
def execute(item, output, attempt, transport=transport, name=name):
|
|
spec = original[item.id]
|
|
# Re-render on placement, including VM fallback: native is always unscaled.
|
|
placed = make_item(spec, args.arm, args.image, deadline_scale=deadline_scale, lane=name, deadline=deadline)
|
|
placed.id = item.id
|
|
record = {**json.loads(placed.inputs["SPEC_URL"]), "lane": name, **transport(placed, output, attempt)}
|
|
if (output / "result.dat").exists():
|
|
key = f"unseen/{placed.input_digest}/{name}/result.dat"
|
|
transport.store._put(key, (output / "result.dat").read_bytes())
|
|
record["result_object"] = key
|
|
return record
|
|
|
|
executors[name] = execute
|
|
return dispatch(items, workload, out, executors, lanes=lanes)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|