mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5.5
26720ed0dc
Committed before the repeats finish. Unit is (change, repeat); paired bootstrap over changes and exact McNemar from openultrasast.stats; failures stay in the denominator. Dry run on the existing single runs reproduces 6/12 and 3/12. Co-Authored-By: Claude Opus 5.5 <[email protected]>
78 lines
3.6 KiB
Python
78 lines
3.6 KiB
Python
"""Paired comparison of the baseline hunt program (DSPy) and the pilot hunter over 5 repeats on the 12 positives.
|
|
|
|
Fixed before the runs finished. Unit = (change, repeat). Arm A = pilot hunter (benchmarks/valid/hunter_arm.py,
|
|
classifier on), arm B = baseline hunt program (openultrasast.hunt.cli). A hit is `benchmarks.unseen.score.catches`.
|
|
Reported: per-change hit rate per arm, overall recall per arm with a Wilson interval over all (change, repeat)
|
|
units, the paired difference B - A with a bootstrap that resamples whole changes (each change is its own
|
|
repository here), and exact McNemar on the (change, repeat) pairs. Instrument failures and parse failures are
|
|
counted and shown, never dropped from the denominator.
|
|
|
|
PYTHONPATH=. .venv/bin/python benchmarks/measurements/2026-10-13-hunt-repeats/analyse.py ~/ousast-results/hunt/repeats
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
from benchmarks.unseen import score
|
|
from openultrasast import stats
|
|
|
|
|
|
def load(root: Path) -> pd.DataFrame:
|
|
rows = []
|
|
for arm, prefix in (("hunter", "hunter-"), ("dspy", "dspy-")):
|
|
for run in sorted(root.glob(prefix + "*")):
|
|
if not run.is_dir() or not (run / "inputs.json").is_file():
|
|
continue
|
|
repeat = int(run.name.split("-")[1])
|
|
for record in score.load_records(run):
|
|
if record.get("kind") != "vulnerability":
|
|
continue
|
|
defects = record.get("push", {}).get("admission", {}).get("defects", [])
|
|
rows.append(
|
|
{
|
|
"arm": arm,
|
|
"repeat": repeat,
|
|
"change": record["id"],
|
|
"repo": record.get("repo"),
|
|
"status": record.get("status", "missing"),
|
|
"parse_failed": bool(record.get("parse_failed")),
|
|
"suspicions": len(defects),
|
|
"hit": int(bool(record.get("done")) and bool(score.catches(record))),
|
|
}
|
|
)
|
|
return pd.DataFrame(rows)
|
|
|
|
|
|
def main(root: Path) -> dict:
|
|
frame = load(root)
|
|
wide = frame.pivot_table(index=["change", "repo", "repeat"], columns="arm", values="hit").dropna().reset_index()
|
|
out: dict = {"units": len(wide), "repeats": sorted(frame["repeat"].unique().tolist())}
|
|
for arm in ("hunter", "dspy"):
|
|
part = frame[frame["arm"] == arm]
|
|
hits, n = int(part["hit"].sum()), len(part)
|
|
out[arm] = {
|
|
"hits": hits,
|
|
"units": n,
|
|
"recall": hits / n if n else None,
|
|
"wilson95": stats.wilson(hits, n),
|
|
"instrument_failures": int((part["status"] == "instrument_failure").sum()),
|
|
"parse_failures": int(part["parse_failed"].sum()),
|
|
"suspicions_mean": float(part["suspicions"].mean()) if n else None,
|
|
}
|
|
estimate = stats.paired_bootstrap(wide, "change", "hunter", "dspy", resamples=5000, seed=0)
|
|
out["dspy_minus_hunter"] = {"diff": estimate.diff, "ci95": estimate.ci, "changes": estimate.groups}
|
|
mc = stats.mcnemar_exact(list(wide["hunter"].astype(bool)), list(wide["dspy"].astype(bool)))
|
|
out["mcnemar"] = {"only_hunter": mc.b, "only_dspy": mc.c, "concordant": mc.concordant, "p": mc.p_value}
|
|
per_change = frame.groupby(["change", "repo", "arm"])["hit"].mean().unstack("arm").reset_index()
|
|
out["per_change"] = per_change.sort_values("repo").to_dict(orient="records")
|
|
return out
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print(json.dumps(main(Path(sys.argv[1]).expanduser()), indent=2, default=str))
|