Open_Ultra_SAST/benchmarks/unseen/replay.py

"""Dispatch frozen unseen changes or a five-change project-history proof."""

from __future__ import annotations

import argparse
import hashlib
import io
import json
import math
import subprocess
import tarfile
import tomllib
from pathlib import Path

from benchmarks.ax.batch import AXLane, DockerLane, Item, Workload, dispatch, write_json
from benchmarks.unseen.score import coverage


def make_item(change, arm, image, *, deadline_scale=1.0, lane="ax", deadline=None):
    if arm not in {"default", "long_deadline"}:
        raise ValueError("unknown replay arm")
    identity = hashlib.sha256(str(change["id"]).encode()).hexdigest()[:14]
    if not math.isfinite(deadline_scale) or deadline_scale <= 0:
        raise ValueError("deadline scale must be finite and positive")
    base = deadline if deadline is not None else 30 if arm == "default" else 300
    applied = round(base * (1 if lane == "docker" else deadline_scale))
    if applied < 1:
        raise ValueError("scaled deadline must be at least one second")
    spec = {
        **change,
        "id": identity,
        "hook_flags": ["--deadline", str(applied)],
        "image": image,
        "base_deadline_seconds": base,
        "deadline_seconds": applied,
        "deadline_scale": deadline_scale,
        "applied_deadline_scale": 1.0 if lane == "docker" else deadline_scale,
    }
    data = json.dumps(spec, sort_keys=True).encode()
    digest = hashlib.sha256(data).hexdigest()
    return Item(
        "replay-" + identity,
        {"SPEC_URL": data},
        "result.dat",
        ("python3", "/app/benchmarks/unseen/replay_entry.py"),
        {},
        digest,
        {"SPEC_URL": "spec.dat"},
    )


def validate_result(data, output):
    with tarfile.open(fileobj=io.BytesIO(data)) as archive:
        members = archive.getmembers()
        required = {"push.json", "push.txt", "engine.json", "instrument.json"}
        if {m.name for m in members} != required or any(not m.isfile() for m in members):
            raise ValueError("replay result requires push, engine and instrument.json")
        payload = {m.name: archive.extractfile(m).read() for m in members}
    instrument = json.loads(payload["instrument.json"])
    push = json.loads(payload["push.json"])
    record = {"instrument": instrument, "push": push}
    record["status"] = coverage(record)
    # Transport/scheduler needs an explicit OOM signal, not a quiet failed replay.
    if "OutOfMemoryError" in json.dumps(push) or "JVM out of memory" in json.dumps(push):
        record["status"] = "oom"
    output.mkdir(parents=True, exist_ok=True)
    for name, body in payload.items():
        (output / name).write_bytes(body)
    (output / "result.dat").write_bytes(data)
    write_json(output / "result.json", record)
    return record


def proof_changes(root):
    """Only the operator's project history; no pool data or fetches."""

    def git(*args):
        return subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True, check=True).stdout.strip()

    changes = []
    for head in git("rev-list", "--no-merges", "--max-count=100", "HEAD", "--", "src").splitlines():
        base = git("rev-parse", head + "^")
        changed = git("diff", "--name-only", base, head).splitlines()
        if not any(name.startswith("src/") and name.endswith(".py") for name in changed):
            continue
        changes.append(
            {
                "id": hashlib.sha256(head.encode()).hexdigest(),
                "repository_url": "https://github.com/norandom/OpenUltraSAST.git",
                "repo": "project-proof",
                "kind": "ordinary",
                "base": base,
                "head": head,
            }
        )
        if len(changes) == 5:
            break
    if len(changes) != 5:
        raise ValueError("history lacks five source changes")
    return changes


def canonical(value):
    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()


def load_slice(path, slice_id, private_path=None):
    if path.suffix != ".toml":
        frozen = json.loads(path.read_text())
        changes = frozen["changes"]
        if frozen["sha256"] != hashlib.sha256(json.dumps(changes, sort_keys=True).encode()).hexdigest():
            raise ValueError("frozen changes digest mismatch")
        return changes
    if not path.name.startswith("pool-"):
        raise ValueError("replay requires a frozen pool, never a draft")
    pool = tomllib.loads(path.read_text())
    freeze = json.loads(path.with_name("freeze-" + path.stem.removeprefix("pool-") + ".json").read_text())
    if hashlib.sha256(canonical(pool)).hexdigest() != freeze["freeze_digest"]:
        raise ValueError("frozen pool digest mismatch")
    private_path = private_path or path.parent / "private" / path.name
    private = {}
    if private_path.exists():
        private = {row["id"]: row for row in tomllib.loads(private_path.read_text()).get("repository", [])}
    changes = []
    for public in pool.get("repository", []):
        if str(public["slice"]) != str(slice_id):
            continue
        row = public
        if public.get("private"):
            row = private.get(public["id"])
            if row is None:
                raise ValueError("missing private pool entry")
            payload = {"url": row["url"], "commits": [[c["base"], c["head"]] for c in row["changes"]]}
            if hashlib.sha256(canonical(payload)).hexdigest() != public["sha256"]:
                raise ValueError("private pool digest mismatch")
        for change in row["changes"]:
            identity = hashlib.sha256(canonical([public["id"], change["kind"], change["base"], change["head"]])).hexdigest()
            changes.append(
                {
                    **change,
                    "id": identity,
                    "repo": public["id"],
                    "repository_url": row["url"],
                    "freeze_digest": freeze["freeze_digest"],
                    "slice": slice_id,
                }
            )
    if not changes:
        raise ValueError("frozen slice has no changes")
    return changes


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--manifest", type=Path, required=True, help="frozen JSON: changes array plus sha256 of canonical changes JSON")
    parser.add_argument("--proof-history", action="store_true", help="write a five-change own-history manifest, then exit")
    parser.add_argument("--image")
    parser.add_argument("--private-manifest", type=Path)
    parser.add_argument("--dry-run", action="store_true")
    parser.add_argument("--arm", choices=("default", "long_deadline"), default="default")
    parser.add_argument("--pool", default="p1")
    parser.add_argument("--slice", default="1")
    parser.add_argument("--out", type=Path)
    parser.add_argument("--lane", choices=("ax", "kind", "docker", "mixed"), default="mixed")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("--deadline-scale", type=float, default=1.0)
    group.add_argument("--calibration", type=Path)
    parser.add_argument("--native-sample", type=int, default=0)
    parser.add_argument("--docker-image-bytes", type=int, help="unpacked image size upper bound, required for VM pull")
    parser.add_argument("--ax-bin", default="ax")
    parser.add_argument("--kubeconfig", type=Path)
    parser.add_argument("--atespace", default="default")
    args = parser.parse_args()
    if args.proof_history:
        changes = proof_changes(Path.cwd())
        write_json(args.manifest, {"changes": changes, "sha256": hashlib.sha256(json.dumps(changes, sort_keys=True).encode()).hexdigest()})
        return 0
    if not args.image:
        parser.error("--image digest is required")
    changes = load_slice(args.manifest, args.slice, args.private_manifest)
    factor = args.deadline_scale
    if args.calibration:
        calibration = json.loads(args.calibration.read_text())
        if calibration.get("status") != "ok" or calibration.get("image") != args.image:
            parser.error("calibration must be successful and match the replay image")
        factor = calibration["quick_total"]["factor"]
    if args.native_sample < 0 or (args.native_sample and args.lane not in {"ax", "kind"}):
        parser.error("--native-sample requires an ax or kind cluster run and a positive K")
    items = [make_item(change, args.arm, args.image, deadline_scale=factor, lane=args.lane) for change in changes]
    if len({i.id for i in items}) != len(items):
        parser.error("duplicate change identity")
    args.out = args.out or Path.home() / "ousast-results/unseen" / args.pool / args.slice / args.arm
    if args.dry_run:
        print(json.dumps({"items": len(items), "input_digest": hashlib.sha256(canonical([i.input_digest for i in items])).hexdigest()}))
        return 0
    if args.out.resolve().is_relative_to(Path(__file__).resolve().parents[2]):
        parser.error("raw replay output must be outside the repository")
    failed = run_replays(changes, args, args.out, args.lane, deadline_scale=factor)
    if args.native_sample:
        failed |= run_replays(sample_changes(changes, args.native_sample), args, args.out / "native", "docker", deadline_scale=factor)
    return 2 if failed else 0


def sample_changes(changes, count):
    if count < 1:
        raise ValueError("sample size must be positive")
    if len({str(c["id"]) for c in changes}) != len(changes):
        raise ValueError("duplicate change identity")
    return sorted(changes, key=lambda c: hashlib.sha256(str(c["id"]).encode()).digest())[:count]


def run_replays(changes, args, out, lane, *, deadline_scale=1.0, deadline=None):
    if out.resolve().is_relative_to(Path(__file__).resolve().parents[2]):
        raise ValueError("raw replay output must be outside the repository")
    items = [make_item(c, args.arm, args.image, deadline_scale=deadline_scale, lane=lane, deadline=deadline) for c in changes]
    write_json(
        out / "inputs.json",
        [{"item_id": item.id, "change": change, "input_digest": item.input_digest} for item, change in zip(items, changes, strict=True)],
    )
    maximum = max(
        max(json.loads(i.inputs["SPEC_URL"])["deadline_seconds"], json.loads(i.inputs["SPEC_URL"])["base_deadline_seconds"]) for i in items
    )
    workload = Workload(args.image, frozenset({"SPEC_URL", "RESULT_URL"}), maximum + 600, validate_result)
    lanes = ("ax", "ax", "docker") if lane == "mixed" else (lane,) * (2 if lane in {"ax", "kind"} else 1)
    original = {item.id: change for item, change in zip(items, changes, strict=True)}
    executors = {}
    for name in set(lanes):
        transport = (DockerLane if name == "docker" else AXLane)(args, workload)

        def execute(item, output, attempt, transport=transport, name=name):
            spec = original[item.id]
            # Re-render on placement, including VM fallback: native is always unscaled.
            placed = make_item(spec, args.arm, args.image, deadline_scale=deadline_scale, lane=name, deadline=deadline)
            placed.id = item.id
            record = {**json.loads(placed.inputs["SPEC_URL"]), "lane": name, **transport(placed, output, attempt)}
            if (output / "result.dat").exists():
                key = f"unseen/{placed.input_digest}/{name}/result.dat"
                transport.store._put(key, (output / "result.dat").read_bytes())
                record["result_object"] = key
            return record

        executors[name] = execute
    return dispatch(items, workload, out, executors, lanes=lanes)


if __name__ == "__main__":
    raise SystemExit(main())