From 6acd3543111350bca684198aa5adcfc90545b7b2 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:24:56 +0200 Subject: [PATCH] feat(independent): fixed-pin recheck and an exploratory scorer for the batched verifier - verify_batched.py --fixed asks the agreed candidates again at the fixed pin (no triage); a function the fix removed is recorded as removed. - score_batched.py applies protocol v2's matching to the batched output (vulnerable pin, and the fixed pin where rechecked) and draws the precision sample. - A finding's site is anchored to the candidate operation asked about; the model's own reported location is kept in `reported_at`. ReactPress's confirmed findAll was reported at the controller file and failed to match its declared site. The scorer anchors older records the same way. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- benchmarks/independent/score_batched.py | 90 ++++++++++++++++++++++++ benchmarks/independent/verify_batched.py | 32 +++++++-- 2 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 benchmarks/independent/score_batched.py diff --git a/benchmarks/independent/score_batched.py b/benchmarks/independent/score_batched.py new file mode 100644 index 0000000..40c8b8c --- /dev/null +++ b/benchmarks/independent/score_batched.py @@ -0,0 +1,90 @@ +"""Exploratory score of a batched-verifier run against the population's cases (protocol v2 matching). + +The batched verifier writes one vulnerable-pin result per case (and a fixed-pin recheck with --fixed), not the +six pins protocol v2 scores, so this applies the same matching rule (`evaluate.matches_v2`) to what exists: +detected when an AGREED finding matches; unstable when only a DISPUTED one does; fixed-side alert when an agreed +finding at the fixed pin matches the fix's new side. The precision sample is every third agreed vulnerable-pin +finding (all if 15 or fewer), to be adjudicated by hand. v2 is spent for tuning: this qualifies nothing. + +Usage: python benchmarks/independent/score_batched.py [--results-suffix batched] [--population population-v2.toml] +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import evaluate # noqa: E402 + + +def anchored(result: dict) -> dict: + """Sites anchored to the candidate asked about (older records used the model's reported location).""" + for key in ("findings", "disputed"): + for f in result.get(key, []): + path, _, function = f["candidate"].partition("::") + site_path, _, rest = f["site"].partition(":") + line = rest.split(":")[0] if site_path == path else "0" + f["site"] = f"{path}:{line}:{function}" + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--population", type=Path, default=HERE / "population-v2.toml") + parser.add_argument("--results-suffix", default="batched") + args = parser.parse_args() + evaluate.use(args.population) + results = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-" + args.results_suffix) + scans = results / "scans" + per_case, pool = [], [] + for case in evaluate.cases(): + path = scans / f"{case['id']}--vulnerable_a.json" + if not path.is_file(): + per_case.append({"id": case["id"], "status": "incomplete"}) + continue + vulnerable = anchored(json.loads(path.read_text())) + old, new = evaluate.hunks(case, "old"), evaluate.hunks(case, "new") + agreed = [f["site"] for f in vulnerable["findings"] if evaluate.matches_v2(f, case, old)] + disputed = [f["site"] for f in vulnerable.get("disputed", []) if evaluate.matches_v2(f, case, old)] + fixed_path = scans / f"{case['id']}--fixed_a.json" + fixed = anchored(json.loads(fixed_path.read_text())) if fixed_path.is_file() else None + fixed_alerts = [f["site"] for f in fixed["findings"] if evaluate.matches_v2(f, case, new)] if fixed else None + per_case.append( + { + "id": case["id"], "family": case["family"], "detected": bool(agreed), "detected_sites": agreed, + "unstable": bool(disputed) and not agreed, "unstable_sites": disputed, + "agreed_findings": len(vulnerable["findings"]), "disputed_findings": len(vulnerable.get("disputed", [])), + "triaged_out": vulnerable.get("triaged_out"), "candidates": vulnerable.get("questions"), + "fixed_side_alerts": fixed_alerts, "fixed_agreed_total": len(fixed["findings"]) if fixed else None, + "removed_by_fix": fixed.get("removed_by_fix") if fixed else None, "usd": vulnerable.get("usd"), "usage": vulnerable.get("usage"), + } + ) # fmt: skip + pool += [{"case": case["id"], **f} for f in vulnerable["findings"]] + pool.sort(key=lambda f: (f["case"], f["site"])) + sample = pool if len(pool) <= 15 else pool[0::3] + scored = [c for c in per_case if c.get("status") != "incomplete"] + detected = sum(c["detected"] for c in scored) + with_fixed = [c for c in scored if c["fixed_side_alerts"] is not None] + result = { + "protocol": "protocol-v2.md matching, exploratory (batched verifier, vulnerable + fixed pins only)", + "cases_scored": len(scored), "cases_incomplete": [c["id"] for c in per_case if c.get("status") == "incomplete"], + "recall": [detected, len(scored)], "recall_wilson95": evaluate.wilson(detected, len(scored)), + "unstable_detections": [c["id"] for c in scored if c["unstable"]], + "fixed_side_alerts": sum(len(c["fixed_side_alerts"]) for c in with_fixed) if with_fixed else None, + "fixed_pins_rechecked": len(with_fixed), + "pooled_agreed_findings": len(pool), "precision_sample_size": len(sample), + "usd": round(sum(float(c.get("usd") or 0) for c in scored), 2), + "cases": per_case, "precision_sample": sample, + } # fmt: skip + (results / "score.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps({k: v for k, v in result.items() if k not in ("cases", "precision_sample")}, indent=1)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/independent/verify_batched.py b/benchmarks/independent/verify_batched.py index fa22749..01dc3ea 100644 --- a/benchmarks/independent/verify_batched.py +++ b/benchmarks/independent/verify_batched.py @@ -122,7 +122,10 @@ def hunt(root: Path, path: str, group: list[list], family: str, model: str, step fn = f.function_name or "" if fn not in names: # attribute by the nearest listed line when the model renamed the function fn = min(group, key=lambda c: abs((c[2] or 0) - (f.line or 0)))[1] - rows.append({"candidate": f"{path}::{fn}", "site": f"{f.path}:{f.line or 0}:{fn}", "family": family, "title": f.title, "witness": f.rationale[:600]}) + # The verdict is about the candidate operation, so the site is the candidate's file and function; the + # model's own location is kept beside it (it often names the file where the input enters instead). + line = f.line if f.path == path else next(c[2] for c in group if c[1] == fn) + rows.append({"candidate": f"{path}::{fn}", "site": f"{path}:{line or 0}:{fn}", "reported_at": f"{f.path}:{f.line or 0}", "family": family, "title": f.title, "witness": f.rationale[:600]}) return rows, recording.usage() @@ -177,6 +180,7 @@ def main() -> int: parser.add_argument("--workers", type=int, default=4) parser.add_argument("--budget-usd", type=float, default=40.0) parser.add_argument("--results-suffix", default="batched") + parser.add_argument("--fixed", action="store_true", help="recheck the AGREED candidates of each case at its fixed pin (no triage)") args = parser.parse_args() load_config() evaluate.use(args.population) @@ -191,10 +195,11 @@ def main() -> int: for case in evaluate.cases(): if only and case["id"] not in only or (subset is not None and case["id"] not in subset): continue - out = out_dir / f"{case['id']}--vulnerable_a.json" + label = "fixed_a" if args.fixed else "vulnerable_a" + out = out_dir / f"{case['id']}--{label}.json" if out.is_file(): continue - log = out_dir / f"{case['id']}--vulnerable_a.jsonl" + log = out_dir / f"{case['id']}--{label}.jsonl" done = {r["path"]: r for r in (json.loads(row) for row in log.read_text().splitlines()) if "unverified" not in r} if log.is_file() else {} spent += sum(float(r.get("usd", 0.0)) for r in done.values()) record = json.loads((candidates_dir / f"{case['id']}.json").read_text()) @@ -202,12 +207,29 @@ def main() -> int: if subset is not None: wanted = {tuple(x) for x in subset[case["id"]]} unique = [c for c in unique if (c[0], c[1]) in wanted] + if args.fixed: + # The agreed findings at the vulnerable pin, by (path, function), asked again at the fixed pin. Line + # numbers are the vulnerable pin's; the file itself is in the prompt. A function the fix removed + # cannot be asked and counts as removed. + vulnerable_out = out_dir / f"{case['id']}--vulnerable_a.json" + if not vulnerable_out.is_file(): + continue + agreed = {tuple(f["candidate"].split("::", 1)) for f in json.loads(vulnerable_out.read_text())["findings"]} + unique = [c for c in unique if (c[0], c[1]) in agreed] + settings = dict(settings, no_triage=True) by_file: dict[str, list[list]] = {} for c in unique: by_file.setdefault(c[0], []).append(c) pending = {p: cs for p, cs in by_file.items() if p not in done} started = time.monotonic() - checkout = evaluate.export(case, "vulnerable_a", case["vulnerable"]) + checkout = evaluate.export(case, label, case["fixed" if args.fixed else "vulnerable"]) + if args.fixed: + present = [c for c in unique if (checkout / c[0]).is_file()] + removed = [f"{c[0]}::{c[1]}" for c in unique if (checkout / c[0]).is_file() is False] + unique, by_file = present, {} + for c in unique: + by_file.setdefault(c[0], []).append(c) + pending = {p: cs for p, cs in by_file.items() if p not in done} account_error = "" try: with concurrent.futures.ProcessPoolExecutor(max_workers=args.workers) as pool, log.open("a") as sink: @@ -245,6 +267,8 @@ def main() -> int: "triaged_out": triaged_out, "passes": args.passes, "usage": usage, "usd": round(sum(float(r.get("usd", 0.0)) for r in done.values()), 4), "seconds": round(time.monotonic() - started, 1), "findings": sorted(agreed, key=lambda f: f["site"]), "disputed": sorted(disputed, key=lambda f: f["site"]), } # fmt: skip + if args.fixed: + result["removed_by_fix"] = removed if complete: out.write_text(json.dumps(result, indent=1) + "\n") print(json.dumps({"case": case["id"], "files": f"{len(done)}/{len(by_file)}", "candidates": len(unique), "triaged_out": triaged_out, "agreed": len(agreed), "disputed": len(disputed), "usd": result["usd"], "total_usd": round(spent, 4), "complete": complete}), flush=True)