mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(independent): fixed-pin recheck and an exploratory scorer for the batched verifier
- verify_batched.py --fixed asks the agreed candidates again at the fixed pin (no triage); a function the fix removed is recorded as removed. - score_batched.py applies protocol v2's matching to the batched output (vulnerable pin, and the fixed pin where rechecked) and draws the precision sample. - A finding's site is anchored to the candidate operation asked about; the model's own reported location is kept in `reported_at`. ReactPress's confirmed findAll was reported at the controller file and failed to match its declared site. The scorer anchors older records the same way. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
072a102b20
commit
6acd354311
@@ -0,0 +1,90 @@
|
||||
"""Exploratory score of a batched-verifier run against the population's cases (protocol v2 matching).
|
||||
|
||||
The batched verifier writes one vulnerable-pin result per case (and a fixed-pin recheck with --fixed), not the
|
||||
six pins protocol v2 scores, so this applies the same matching rule (`evaluate.matches_v2`) to what exists:
|
||||
detected when an AGREED finding matches; unstable when only a DISPUTED one does; fixed-side alert when an agreed
|
||||
finding at the fixed pin matches the fix's new side. The precision sample is every third agreed vulnerable-pin
|
||||
finding (all if 15 or fewer), to be adjudicated by hand. v2 is spent for tuning: this qualifies nothing.
|
||||
|
||||
Usage: python benchmarks/independent/score_batched.py [--results-suffix batched] [--population population-v2.toml]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
import evaluate # noqa: E402
|
||||
|
||||
|
||||
def anchored(result: dict) -> dict:
|
||||
"""Sites anchored to the candidate asked about (older records used the model's reported location)."""
|
||||
for key in ("findings", "disputed"):
|
||||
for f in result.get(key, []):
|
||||
path, _, function = f["candidate"].partition("::")
|
||||
site_path, _, rest = f["site"].partition(":")
|
||||
line = rest.split(":")[0] if site_path == path else "0"
|
||||
f["site"] = f"{path}:{line}:{function}"
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--population", type=Path, default=HERE / "population-v2.toml")
|
||||
parser.add_argument("--results-suffix", default="batched")
|
||||
args = parser.parse_args()
|
||||
evaluate.use(args.population)
|
||||
results = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-" + args.results_suffix)
|
||||
scans = results / "scans"
|
||||
per_case, pool = [], []
|
||||
for case in evaluate.cases():
|
||||
path = scans / f"{case['id']}--vulnerable_a.json"
|
||||
if not path.is_file():
|
||||
per_case.append({"id": case["id"], "status": "incomplete"})
|
||||
continue
|
||||
vulnerable = anchored(json.loads(path.read_text()))
|
||||
old, new = evaluate.hunks(case, "old"), evaluate.hunks(case, "new")
|
||||
agreed = [f["site"] for f in vulnerable["findings"] if evaluate.matches_v2(f, case, old)]
|
||||
disputed = [f["site"] for f in vulnerable.get("disputed", []) if evaluate.matches_v2(f, case, old)]
|
||||
fixed_path = scans / f"{case['id']}--fixed_a.json"
|
||||
fixed = anchored(json.loads(fixed_path.read_text())) if fixed_path.is_file() else None
|
||||
fixed_alerts = [f["site"] for f in fixed["findings"] if evaluate.matches_v2(f, case, new)] if fixed else None
|
||||
per_case.append(
|
||||
{
|
||||
"id": case["id"], "family": case["family"], "detected": bool(agreed), "detected_sites": agreed,
|
||||
"unstable": bool(disputed) and not agreed, "unstable_sites": disputed,
|
||||
"agreed_findings": len(vulnerable["findings"]), "disputed_findings": len(vulnerable.get("disputed", [])),
|
||||
"triaged_out": vulnerable.get("triaged_out"), "candidates": vulnerable.get("questions"),
|
||||
"fixed_side_alerts": fixed_alerts, "fixed_agreed_total": len(fixed["findings"]) if fixed else None,
|
||||
"removed_by_fix": fixed.get("removed_by_fix") if fixed else None, "usd": vulnerable.get("usd"), "usage": vulnerable.get("usage"),
|
||||
}
|
||||
) # fmt: skip
|
||||
pool += [{"case": case["id"], **f} for f in vulnerable["findings"]]
|
||||
pool.sort(key=lambda f: (f["case"], f["site"]))
|
||||
sample = pool if len(pool) <= 15 else pool[0::3]
|
||||
scored = [c for c in per_case if c.get("status") != "incomplete"]
|
||||
detected = sum(c["detected"] for c in scored)
|
||||
with_fixed = [c for c in scored if c["fixed_side_alerts"] is not None]
|
||||
result = {
|
||||
"protocol": "protocol-v2.md matching, exploratory (batched verifier, vulnerable + fixed pins only)",
|
||||
"cases_scored": len(scored), "cases_incomplete": [c["id"] for c in per_case if c.get("status") == "incomplete"],
|
||||
"recall": [detected, len(scored)], "recall_wilson95": evaluate.wilson(detected, len(scored)),
|
||||
"unstable_detections": [c["id"] for c in scored if c["unstable"]],
|
||||
"fixed_side_alerts": sum(len(c["fixed_side_alerts"]) for c in with_fixed) if with_fixed else None,
|
||||
"fixed_pins_rechecked": len(with_fixed),
|
||||
"pooled_agreed_findings": len(pool), "precision_sample_size": len(sample),
|
||||
"usd": round(sum(float(c.get("usd") or 0) for c in scored), 2),
|
||||
"cases": per_case, "precision_sample": sample,
|
||||
} # fmt: skip
|
||||
(results / "score.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
print(json.dumps({k: v for k, v in result.items() if k not in ("cases", "precision_sample")}, indent=1))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -122,7 +122,10 @@ def hunt(root: Path, path: str, group: list[list], family: str, model: str, step
|
||||
fn = f.function_name or ""
|
||||
if fn not in names: # attribute by the nearest listed line when the model renamed the function
|
||||
fn = min(group, key=lambda c: abs((c[2] or 0) - (f.line or 0)))[1]
|
||||
rows.append({"candidate": f"{path}::{fn}", "site": f"{f.path}:{f.line or 0}:{fn}", "family": family, "title": f.title, "witness": f.rationale[:600]})
|
||||
# The verdict is about the candidate operation, so the site is the candidate's file and function; the
|
||||
# model's own location is kept beside it (it often names the file where the input enters instead).
|
||||
line = f.line if f.path == path else next(c[2] for c in group if c[1] == fn)
|
||||
rows.append({"candidate": f"{path}::{fn}", "site": f"{path}:{line or 0}:{fn}", "reported_at": f"{f.path}:{f.line or 0}", "family": family, "title": f.title, "witness": f.rationale[:600]})
|
||||
return rows, recording.usage()
|
||||
|
||||
|
||||
@@ -177,6 +180,7 @@ def main() -> int:
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
parser.add_argument("--budget-usd", type=float, default=40.0)
|
||||
parser.add_argument("--results-suffix", default="batched")
|
||||
parser.add_argument("--fixed", action="store_true", help="recheck the AGREED candidates of each case at its fixed pin (no triage)")
|
||||
args = parser.parse_args()
|
||||
load_config()
|
||||
evaluate.use(args.population)
|
||||
@@ -191,10 +195,11 @@ def main() -> int:
|
||||
for case in evaluate.cases():
|
||||
if only and case["id"] not in only or (subset is not None and case["id"] not in subset):
|
||||
continue
|
||||
out = out_dir / f"{case['id']}--vulnerable_a.json"
|
||||
label = "fixed_a" if args.fixed else "vulnerable_a"
|
||||
out = out_dir / f"{case['id']}--{label}.json"
|
||||
if out.is_file():
|
||||
continue
|
||||
log = out_dir / f"{case['id']}--vulnerable_a.jsonl"
|
||||
log = out_dir / f"{case['id']}--{label}.jsonl"
|
||||
done = {r["path"]: r for r in (json.loads(row) for row in log.read_text().splitlines()) if "unverified" not in r} if log.is_file() else {}
|
||||
spent += sum(float(r.get("usd", 0.0)) for r in done.values())
|
||||
record = json.loads((candidates_dir / f"{case['id']}.json").read_text())
|
||||
@@ -202,12 +207,29 @@ def main() -> int:
|
||||
if subset is not None:
|
||||
wanted = {tuple(x) for x in subset[case["id"]]}
|
||||
unique = [c for c in unique if (c[0], c[1]) in wanted]
|
||||
if args.fixed:
|
||||
# The agreed findings at the vulnerable pin, by (path, function), asked again at the fixed pin. Line
|
||||
# numbers are the vulnerable pin's; the file itself is in the prompt. A function the fix removed
|
||||
# cannot be asked and counts as removed.
|
||||
vulnerable_out = out_dir / f"{case['id']}--vulnerable_a.json"
|
||||
if not vulnerable_out.is_file():
|
||||
continue
|
||||
agreed = {tuple(f["candidate"].split("::", 1)) for f in json.loads(vulnerable_out.read_text())["findings"]}
|
||||
unique = [c for c in unique if (c[0], c[1]) in agreed]
|
||||
settings = dict(settings, no_triage=True)
|
||||
by_file: dict[str, list[list]] = {}
|
||||
for c in unique:
|
||||
by_file.setdefault(c[0], []).append(c)
|
||||
pending = {p: cs for p, cs in by_file.items() if p not in done}
|
||||
started = time.monotonic()
|
||||
checkout = evaluate.export(case, "vulnerable_a", case["vulnerable"])
|
||||
checkout = evaluate.export(case, label, case["fixed" if args.fixed else "vulnerable"])
|
||||
if args.fixed:
|
||||
present = [c for c in unique if (checkout / c[0]).is_file()]
|
||||
removed = [f"{c[0]}::{c[1]}" for c in unique if (checkout / c[0]).is_file() is False]
|
||||
unique, by_file = present, {}
|
||||
for c in unique:
|
||||
by_file.setdefault(c[0], []).append(c)
|
||||
pending = {p: cs for p, cs in by_file.items() if p not in done}
|
||||
account_error = ""
|
||||
try:
|
||||
with concurrent.futures.ProcessPoolExecutor(max_workers=args.workers) as pool, log.open("a") as sink:
|
||||
@@ -245,6 +267,8 @@ def main() -> int:
|
||||
"triaged_out": triaged_out, "passes": args.passes, "usage": usage, "usd": round(sum(float(r.get("usd", 0.0)) for r in done.values()), 4),
|
||||
"seconds": round(time.monotonic() - started, 1), "findings": sorted(agreed, key=lambda f: f["site"]), "disputed": sorted(disputed, key=lambda f: f["site"]),
|
||||
} # fmt: skip
|
||||
if args.fixed:
|
||||
result["removed_by_fix"] = removed
|
||||
if complete:
|
||||
out.write_text(json.dumps(result, indent=1) + "\n")
|
||||
print(json.dumps({"case": case["id"], "files": f"{len(done)}/{len(by_file)}", "candidates": len(unique), "triaged_out": triaged_out, "agreed": len(agreed), "disputed": len(disputed), "usd": result["usd"], "total_usd": round(spent, 4), "complete": complete}), flush=True)
|
||||
|
||||
Reference in New Issue
Block a user