mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
feat(hunt): candidates scored on folds through the runner; the pilot gain, once per accepted program (6.1)
rounds.py runs a candidate over compile or held-out folds through runner.run, so a tool-using
program has its tools bound to each change's checkout, and turns paired records into the
FoldResults accept.decide reads (closes the gap noted in bacd7a1). report.py runs an accepted
program once over the held-out pilot: a ledger line reserved before the run refuses a second run
per program id, contaminated or unchecked training runs are refused, the instrument is proved per
change and more than 5% failures makes the run invalid with no numbers quoted, the pilot's own
scorer and metrics give the after numbers, the recorded hunter numbers give the before, baseline
repeats pair with the single run, the proof arm's workload per 100 pushes is counted, and the gate
distance is read off the confidence curve. 124 hunt and stats tests pass. Implemented by Codex
(gpt-6-astra), gated here.
Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
bacd7a1e3f
commit
6f9613329e
@@ -0,0 +1,472 @@
|
||||
"""Report one accepted program on the held-out pilot, with instrument proof before scores."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from dataclasses import asdict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .. import stats
|
||||
from . import accept, contamination, runner
|
||||
from .ablation import _arm_summary, _findings, _write
|
||||
from .metric import search_outcome
|
||||
from .propose import Candidate
|
||||
from .rounds import _candidate_program, _estimate, _families, _outcome, _paired
|
||||
|
||||
|
||||
def _reserve(root: Path, program_id: str) -> Path:
|
||||
"""Reserve a program under a file lock and flush the ledger before starting its run."""
|
||||
date = datetime.now(UTC).date().isoformat()
|
||||
if not program_id or any(c not in "0123456789abcdef" for c in program_id):
|
||||
raise ValueError("program ID must be a hexadecimal content address")
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
directory = root / f"{date}-hunt-gain-{program_id[:12]}"
|
||||
with (root / "hunt-pilot-runs.jsonl").open("a+", encoding="utf-8") as stream:
|
||||
fcntl.flock(stream, fcntl.LOCK_EX)
|
||||
stream.seek(0)
|
||||
if any(json.loads(line)["program_id"] == program_id for line in stream if line.strip()):
|
||||
raise ValueError("this program ID already has a pilot run")
|
||||
directory.mkdir(exist_ok=False)
|
||||
stream.write(json.dumps(dict(program_id=program_id, date=date, measurement_directory=str(directory.resolve()))) + "\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
return directory
|
||||
|
||||
|
||||
def _records(directory: Path, changes: Sequence[Mapping]) -> list[dict]:
|
||||
"""Load benchmark records and retain every expected change, even after a budget stop."""
|
||||
from benchmarks.unseen import score
|
||||
|
||||
records = score.load_records(Path(directory)) if (Path(directory) / "inputs.json").exists() else []
|
||||
indexed = {r["id"]: r for r in records}
|
||||
if len(indexed) != len(records) or set(indexed) - {c["id"] for c in changes}:
|
||||
raise ValueError("records do not match the pilot population")
|
||||
result = []
|
||||
for change in changes:
|
||||
record = indexed.get(change["id"], dict(change))
|
||||
if any(record.get(key) != change.get(key) for key in ("repo", "base", "head", "kind", "family", "files", "targets")):
|
||||
raise ValueError("record describes a different pilot change")
|
||||
result.append(record)
|
||||
return result
|
||||
|
||||
|
||||
def _instrument(records: Sequence[Mapping]) -> dict:
|
||||
"""Record changed-file bytes and verified revisions; unreadable and absent changes fail."""
|
||||
from benchmarks.unseen import score
|
||||
|
||||
proof = []
|
||||
for record in records:
|
||||
instrument = record.get("instrument", {})
|
||||
sizes = instrument.get("changed_bytes", {})
|
||||
failed = not record.get("done") or record.get("status") == "instrument_failure" or score.coverage(record) == "instrument_failure"
|
||||
proof.append(
|
||||
dict(
|
||||
id=record["id"],
|
||||
repo=record["repo"],
|
||||
base=record["base"],
|
||||
head=record["head"],
|
||||
base_verified=bool(instrument.get("base_verified")),
|
||||
head_verified=bool(instrument.get("head_verified")),
|
||||
changed_bytes={f["path"]: sizes.get(f["path"], 0) for f in record.get("files", [])},
|
||||
instrument=dict(instrument),
|
||||
instrument_failure=failed,
|
||||
reason=instrument.get("failure") or ("missing or unreadable record" if failed else None),
|
||||
)
|
||||
)
|
||||
failures = sum(row["instrument_failure"] for row in proof)
|
||||
return dict(
|
||||
changes=len(proof),
|
||||
failures=failures,
|
||||
failure_fraction=failures / len(proof) if proof else None,
|
||||
valid=bool(proof) and failures / len(proof) <= 0.05,
|
||||
records=proof,
|
||||
)
|
||||
|
||||
|
||||
def _clean(records: Sequence[Mapping]) -> list[dict]:
|
||||
"""Remove stale findings on failures while preserving the scorer's coverage checks."""
|
||||
from benchmarks.unseen import score
|
||||
|
||||
result = []
|
||||
for original in records:
|
||||
record = deepcopy(original)
|
||||
if not record.get("done") or record.get("status") == "instrument_failure" or score.coverage(record) == "instrument_failure":
|
||||
record["status"] = "instrument_failure"
|
||||
record.setdefault("instrument", {})["failure"] = "instrument_failure"
|
||||
outcome = _outcome(record, record, 10)
|
||||
if not record.get("done") or record.get("status") == "instrument_failure" or record.get("parse_failed"):
|
||||
record["push"] = {"admission": {"defects": []}, "quick_tier": []}
|
||||
else:
|
||||
record.setdefault("push", {}).setdefault("admission", {})["defects"] = outcome["defects"]
|
||||
result.append(record)
|
||||
return result
|
||||
|
||||
|
||||
def _rate(values: list[float], repos: list[str], *, binary: bool = False, scale: float = 1) -> dict:
|
||||
"""Report a mean with its count and a repository-clustered interval, plus Wilson for proportions."""
|
||||
from benchmarks.unseen import score
|
||||
|
||||
interval = score.cluster_interval(list(zip(repos, values, strict=True)), seed=0)
|
||||
result = dict(
|
||||
rate=scale * sum(values) / len(values) if values else None,
|
||||
count=sum(values),
|
||||
n=len(values),
|
||||
groups=len(set(repos)),
|
||||
ci=[scale * v for v in interval] if values else None,
|
||||
)
|
||||
if binary:
|
||||
wilson = stats.wilson(int(sum(values)), len(values))
|
||||
result["wilson95"] = [scale * v for v in wilson] if wilson else None
|
||||
return result
|
||||
|
||||
|
||||
def _family_records(records: Sequence[Mapping], family: str | None) -> list[dict]:
|
||||
"""Keep a family's positives and count only its findings over all ordinary pushes."""
|
||||
result = []
|
||||
for original in records:
|
||||
if family is not None and original["kind"] == "vulnerability" and original.get("family") != family:
|
||||
continue
|
||||
record = deepcopy(original)
|
||||
admission = record.setdefault("push", {}).setdefault("admission", {})
|
||||
admission["defects"] = [d for d in admission.get("defects", []) if family is None or d.get("family") == family]
|
||||
result.append(record)
|
||||
return result
|
||||
|
||||
|
||||
def _summaries(records: list[dict], families: list[str]) -> dict:
|
||||
"""Use the pilot functions for push metrics and the hunt metric for search outcomes."""
|
||||
from benchmarks.unseen import score
|
||||
from benchmarks.valid import metrics
|
||||
|
||||
summaries = {}
|
||||
for family in ["pooled", *families]:
|
||||
part = _family_records(records, None if family == "pooled" else family)
|
||||
positives = [r for r in part if r["kind"] == "vulnerability"]
|
||||
ordinary = [r for r in part if r["kind"] == "ordinary"]
|
||||
outcomes = {r["id"]: search_outcome(r, _findings(r.get("push", {}).get("admission", {}).get("defects", []))) for r in part}
|
||||
search = dict(
|
||||
recall=_rate([int(outcomes[r["id"]].hit) for r in positives], [r["repo"] for r in positives], binary=True),
|
||||
suspicions_per_push=_rate([outcomes[r["id"]].suspicions for r in ordinary], [r["repo"] for r in ordinary]),
|
||||
)
|
||||
# The whole run already passed its instrument gate. The pilot scorer excludes these failures
|
||||
# from push denominators, while the search summaries above keep them as misses.
|
||||
usable = [r for r in part if score.coverage(r) != "instrument_failure"]
|
||||
if usable:
|
||||
scored = score.score(metrics.scoring_records(usable))
|
||||
push = metrics.summarize(usable, scored=scored)
|
||||
else:
|
||||
push = {key: score.rate([]) for key in ("recall", "precision", "false_alarms_per_100_pushes")}
|
||||
summaries[family] = dict(search=search, push=push)
|
||||
return summaries
|
||||
|
||||
|
||||
def _curve(records: list[dict]) -> tuple[list[dict], dict]:
|
||||
"""Extend ablation confidence cuts with false alarms, intervals and the two gate bounds."""
|
||||
from benchmarks.unseen import score
|
||||
|
||||
units = [dict(change=r, after=_outcome(r, r, 10)) for r in records]
|
||||
curve = _arm_summary(units, "after", 10)["curve"]
|
||||
for point in curve:
|
||||
cut = point["confidence"]
|
||||
filtered = deepcopy(records)
|
||||
for record in filtered:
|
||||
defects = record.get("push", {}).get("admission", {}).get("defects", [])
|
||||
record.setdefault("push", {}).setdefault("admission", {})["defects"] = [
|
||||
d
|
||||
for d in defects
|
||||
if isinstance(d.get("confidence"), int | float) and math.isfinite(d["confidence"]) and d["confidence"] >= cut
|
||||
]
|
||||
positives = [r for r in filtered if r["kind"] == "vulnerability"]
|
||||
ordinary = [r for r in filtered if r["kind"] == "ordinary"]
|
||||
hits = [int(search_outcome(r, _findings(r["push"]["admission"]["defects"])).hit) for r in positives]
|
||||
point["estimates"] = dict(
|
||||
recall=_rate(hits, [r["repo"] for r in positives], binary=True),
|
||||
suspicions_per_push=_rate([len(r["push"]["admission"]["defects"]) for r in ordinary], [r["repo"] for r in ordinary]),
|
||||
false_alarms_per_100_pushes=_rate(
|
||||
[int(score.false_alarm(r)) for r in ordinary], [r["repo"] for r in ordinary], binary=True, scale=100
|
||||
),
|
||||
)
|
||||
point["false_alarms_per_100_pushes"] = point["estimates"]["false_alarms_per_100_pushes"]["rate"]
|
||||
high_recall = [p for p in curve if p["recall"] is not None and p["recall"] >= 0.9 and p["false_alarms_per_100_pushes"] is not None]
|
||||
low_alarm = [
|
||||
p
|
||||
for p in curve
|
||||
if p["false_alarms_per_100_pushes"] is not None and p["false_alarms_per_100_pushes"] < 10 and p["recall"] is not None
|
||||
]
|
||||
return curve, dict(
|
||||
false_alarms_at_recall_90=min(high_recall, key=lambda p: p["false_alarms_per_100_pushes"]) if high_recall else None,
|
||||
recall_under_10_false_alarms=max(low_alarm, key=lambda p: p["recall"]) if low_alarm else None,
|
||||
)
|
||||
|
||||
|
||||
def _precision(column: Any) -> float | None:
|
||||
"""Reduce paired (correctly flagged, flagged) counts to push precision."""
|
||||
flagged = sum(value[1] for value in column)
|
||||
return sum(value[0] for value in column) / flagged if flagged else None
|
||||
|
||||
|
||||
def _baseline_comparison(baselines: list[list[dict]], records: list[dict], families: list[str]) -> dict:
|
||||
"""Pair every baseline repeat with the one accepted run, without inventing new repetitions."""
|
||||
from benchmarks.unseen import score
|
||||
from benchmarks.valid import metrics
|
||||
|
||||
units = [
|
||||
dict(change=after, repeat=repeat, a=_outcome(before, before, 10), b=_outcome(after, after, 10))
|
||||
for repeat, baseline in enumerate(baselines)
|
||||
for before, after in zip(baseline, records, strict=True)
|
||||
]
|
||||
result = {}
|
||||
for name in ["pooled", *families]:
|
||||
family = None if name == "pooled" else name
|
||||
recall, suspicions = _paired(units, family, seed=0, resamples=stats.RESAMPLES)
|
||||
positives = [u for u in units if u["change"]["kind"] == "vulnerability" and (family is None or u["change"].get("family") == family)]
|
||||
paired = dict(
|
||||
search_recall=asdict(recall),
|
||||
suspicions_per_push=asdict(suspicions),
|
||||
mcnemar=asdict(stats.mcnemar_exact([u["a"]["hit"] for u in positives], [u["b"]["hit"] for u in positives])),
|
||||
)
|
||||
rows = {"recall": [], "false_alarms_per_100_pushes": [], "precision": []}
|
||||
for baseline in baselines:
|
||||
before = metrics.scoring_records(_family_records(baseline, family))
|
||||
after = metrics.scoring_records(_family_records(records, family))
|
||||
for a, b in zip(before, after, strict=True):
|
||||
flags = [int(score.false_alarm(r)) for r in (a, b)]
|
||||
hits = [int(bool(score.catches(r))) if r["kind"] == "vulnerability" else 0 for r in (a, b)]
|
||||
key = "recall" if a["kind"] == "vulnerability" else "false_alarms_per_100_pushes"
|
||||
values = hits if key == "recall" else [100 * v for v in flags]
|
||||
rows[key].append(dict(repo=a["repo"], a=values[0], b=values[1]))
|
||||
rows["precision"].append(dict(repo=a["repo"], a=(hits[0], flags[0]), b=(hits[1], flags[1])))
|
||||
for metric, frame in rows.items():
|
||||
paired[metric] = asdict(_estimate(frame, seed=0, resamples=stats.RESAMPLES, stat=_precision if metric == "precision" else None))
|
||||
for metric in ("recall", "false_alarms_per_100_pushes"):
|
||||
paired[metric + "_mcnemar"] = asdict(
|
||||
stats.mcnemar_exact([bool(r["a"]) for r in rows[metric]], [bool(r["b"]) for r in rows[metric]])
|
||||
)
|
||||
result[name] = paired
|
||||
return result
|
||||
|
||||
|
||||
def _before(pilot_metrics: Mapping[str, Path]) -> dict:
|
||||
"""Copy the two recorded metric files unchanged, with their source paths."""
|
||||
if set(pilot_metrics) != {"raw", "filtered"}:
|
||||
raise ValueError("pilot_metrics must map raw and filtered to their recorded JSON files")
|
||||
return {name: dict(source=str(Path(path)), metrics=json.loads(Path(path).read_text())) for name, path in pilot_metrics.items()}
|
||||
|
||||
|
||||
def _format(metric: Mapping | None) -> str:
|
||||
"""Show the estimate, interval and count together, leaving missing measurements explicit."""
|
||||
if not metric or metric.get("rate") is None:
|
||||
return "not recorded" if not metric else f"not measured (n={metric.get('n', 0)})"
|
||||
intervals = []
|
||||
for key, label in (("ci", "cluster 95%"), ("cluster95", "cluster 95%"), ("wilson95", "Wilson 95%")):
|
||||
ci = metric.get(key)
|
||||
if ci and all(v is not None for v in ci):
|
||||
intervals.append(f"{label} {ci[0]:.6g} to {ci[1]:.6g}")
|
||||
return f"{metric['rate']:.6g}; {'; '.join(intervals)}; count={metric['count']}, n={metric['n']}"
|
||||
|
||||
|
||||
def _notes(analysis: dict) -> str:
|
||||
"""Write readable before-and-after tables and the evidence needed to interpret them."""
|
||||
lines = ["# Hunter pilot gain", "", f"Program: `{analysis['program_id']}`", ""]
|
||||
proof = analysis["instrument"]
|
||||
lines += [
|
||||
"## Instrument",
|
||||
"",
|
||||
f"{proof['failures']} instrument failures across {proof['changes']} changes.",
|
||||
"Bytes per changed file and verified base and head are in instrument.json.",
|
||||
"",
|
||||
]
|
||||
if not analysis["valid"]:
|
||||
lines += [f"Run invalid: {analysis['reason']}. No performance numbers are quoted.", ""]
|
||||
else:
|
||||
lines += [
|
||||
"## Before and after",
|
||||
"",
|
||||
"Before values are quoted as recorded; unavailable family or search values are not inferred.",
|
||||
"Recall and precision are proportions. False alarms are per 100 ordinary pushes.",
|
||||
"Push metrics exclude instrument failures as in the pilot. Search metrics and paired comparisons retain failures as misses.",
|
||||
"The confidence curve describes the search output; it does not choose a threshold or change the classifier.",
|
||||
"",
|
||||
"| Family | Metric | Before raw | Before filtered | After |",
|
||||
"|---|---|---|---|---|",
|
||||
]
|
||||
for family, summary in analysis["after"].items():
|
||||
for stage, key, label in (
|
||||
("search", "recall", "Search recall"),
|
||||
("search", "suspicions_per_push", "Suspicions per ordinary push"),
|
||||
("push", "recall", "Push recall"),
|
||||
("push", "false_alarms_per_100_pushes", "False alarms per 100"),
|
||||
("push", "precision", "Precision"),
|
||||
):
|
||||
before = [
|
||||
analysis["before"][name]["metrics"].get(key) if family == "pooled" and stage == "push" else None
|
||||
for name in ("raw", "filtered")
|
||||
]
|
||||
lines.append(f"| {family} | {label} | {_format(before[0])} | {_format(before[1])} | {_format(summary[stage][key])} |")
|
||||
for name, recorded in analysis["before"].items():
|
||||
lines += ["", f"Recorded {name} source: `{recorded['source']}`."]
|
||||
if analysis.get("baseline"):
|
||||
lines += [
|
||||
"",
|
||||
"## Baseline repeats",
|
||||
"",
|
||||
"Each repeat is paired with the single accepted run; intervals resample repositories.",
|
||||
"",
|
||||
"| Repeat | Family | Metric | Baseline | After |",
|
||||
"|---|---|---|---|---|",
|
||||
]
|
||||
for repeat, baseline in enumerate(analysis["baseline"]):
|
||||
if not baseline["valid"]:
|
||||
lines += [f"Baseline repeat {repeat} is invalid; its performance numbers and paired gain are not quoted."]
|
||||
continue
|
||||
for family, summary in baseline["summary"].items():
|
||||
for stage, keys in (
|
||||
("search", ("recall", "suspicions_per_push")),
|
||||
("push", ("recall", "false_alarms_per_100_pushes", "precision")),
|
||||
):
|
||||
for key in keys:
|
||||
lines.append(
|
||||
f"| {repeat} | {family} | {stage} {key} | {_format(summary[stage][key])} | "
|
||||
f"{_format(analysis['after'][family][stage][key])} |"
|
||||
)
|
||||
if "paired" in analysis:
|
||||
lines += [
|
||||
"",
|
||||
"Paired differences and exact McNemar results:",
|
||||
"",
|
||||
"```json",
|
||||
json.dumps(analysis["paired"], indent=2),
|
||||
"```",
|
||||
]
|
||||
lines += ["", "## Workload", ""]
|
||||
for name, value in analysis["workload"].items():
|
||||
lines.append(f"{name.replace('_', ' ')}: {_format(value)}.")
|
||||
lines += ["", "## Gate distance", ""]
|
||||
for name, point in analysis["gate_distance"].items():
|
||||
label = (
|
||||
"Lowest false alarms at recall at least 0.9"
|
||||
if name == "false_alarms_at_recall_90"
|
||||
else "Highest recall under 10 false alarms per 100"
|
||||
)
|
||||
lines.append(
|
||||
f"{label}: "
|
||||
+ (
|
||||
"the curve never reaches this gate bound."
|
||||
if point is None
|
||||
else f"recall {_format(point['estimates']['recall'])}; "
|
||||
f"false alarms {_format(point['estimates']['false_alarms_per_100_pushes'])}."
|
||||
)
|
||||
)
|
||||
lines += ["", "## Acceptance comparison", "", "```json", json.dumps(analysis["comparison"], indent=2), "```", ""]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def pilot_gain(
|
||||
candidate: Candidate,
|
||||
decision: accept.Decision,
|
||||
*,
|
||||
pilot_manifest: Path,
|
||||
repos: Mapping[str, str],
|
||||
training_runs: Sequence[Path],
|
||||
out_root: Path,
|
||||
lm: Any,
|
||||
budget: Mapping[str, float],
|
||||
baseline_runs: Sequence[Path] = (),
|
||||
pilot_metrics: Mapping[str, Path],
|
||||
tools_for: Any = None,
|
||||
load_changes: Any = None,
|
||||
) -> dict:
|
||||
"""Run an accepted program once and write analysis.json, instrument.json and NOTES.md.
|
||||
|
||||
pilot_metrics maps raw and filtered to the original metric JSON files. Baseline runs are
|
||||
single-repeat pilot record directories, each paired with this one accepted run. No command
|
||||
line scorers or real model clients are constructed here. Budget exhaustion or more than five
|
||||
percent instrument failures leaves an invalid measurement without performance numbers.
|
||||
"""
|
||||
import dspy
|
||||
|
||||
from benchmarks.unseen import score
|
||||
from benchmarks.unseen.replay import load_slice
|
||||
|
||||
from .program import repository_tools
|
||||
|
||||
if not decision.accepted or decision.comparison.get("candidate_id") != candidate.program_id:
|
||||
raise ValueError("pilot gain requires the accepted decision for this candidate")
|
||||
training_runs = [Path(p) for p in training_runs]
|
||||
if contamination.clean_runs(training_runs) != training_runs:
|
||||
raise ValueError("contaminated training runs cannot report pilot gain")
|
||||
before = _before(pilot_metrics)
|
||||
program = _candidate_program(candidate, tools_for)
|
||||
changes = list((load_changes or load_slice)(Path(pilot_manifest), "1"))
|
||||
if not changes or len({c["id"] for c in changes}) != len(changes):
|
||||
raise ValueError("pilot changes must be nonempty and have unique IDs")
|
||||
directory = _reserve(Path(out_root), candidate.program_id)
|
||||
with dspy.context(lm=lm, adapter=dspy.ChatAdapter()):
|
||||
artifact = runner.run(
|
||||
Path(pilot_manifest),
|
||||
repos,
|
||||
directory / "records",
|
||||
program=program,
|
||||
program_id=candidate.program_id,
|
||||
mask=candidate.mask,
|
||||
pilot_report=True,
|
||||
only="all",
|
||||
budget=budget,
|
||||
tools_for=tools_for or repository_tools,
|
||||
load_changes=lambda _path, _slice: changes,
|
||||
versions={"dspy": dspy.__version__},
|
||||
)
|
||||
records = _records(directory / "records", changes)
|
||||
proof = _instrument(records)
|
||||
_write(directory / "instrument.json", proof)
|
||||
analysis = dict(
|
||||
program_id=candidate.program_id,
|
||||
comparison=decision.comparison,
|
||||
measurement_directory=directory.name,
|
||||
valid=proof["valid"] and artifact is not None,
|
||||
instrument={k: v for k, v in proof.items() if k != "records"},
|
||||
)
|
||||
if not analysis["valid"]:
|
||||
analysis["reason"] = "more than 5% instrument failures" if not proof["valid"] else "runner budget exhausted"
|
||||
else:
|
||||
records = _clean(records)
|
||||
# Run the same overall instrument gate and scorer as the pilot before quoting any scores.
|
||||
score.score(records)
|
||||
families = _families([dict(change=r, after=_outcome(r, r, 10)) for r in records], ("after",))
|
||||
baseline_records = [_clean(_records(Path(path), changes)) for path in baseline_runs]
|
||||
for baseline in baseline_records:
|
||||
families = sorted(set(families) | set(_families([dict(change=r, before=_outcome(r, r, 10)) for r in baseline], ("before",))))
|
||||
curve, distance = _curve(records)
|
||||
counts = [search_outcome(r, _findings(r["push"]["admission"]["defects"])).suspicions for r in records]
|
||||
analysis.update(
|
||||
before=before,
|
||||
after=_summaries(records, families),
|
||||
curve=curve,
|
||||
gate_distance=distance,
|
||||
workload=dict(
|
||||
suspicions_per_100_pushes=_rate(counts, [r["repo"] for r in records], scale=100),
|
||||
flagged_pushes_per_100_pushes=_rate([int(n > 0) for n in counts], [r["repo"] for r in records], binary=True, scale=100),
|
||||
),
|
||||
)
|
||||
if baseline_records:
|
||||
analysis["baseline"] = []
|
||||
for path, baseline in zip(baseline_runs, baseline_records, strict=True):
|
||||
instrument = _instrument(baseline)
|
||||
entry = dict(directory=str(path), valid=instrument["valid"], instrument=instrument)
|
||||
if entry["valid"]:
|
||||
entry["summary"] = _summaries(baseline, families)
|
||||
analysis["baseline"].append(entry)
|
||||
if all(row["valid"] for row in analysis["baseline"]):
|
||||
analysis["paired"] = _baseline_comparison(baseline_records, records, families)
|
||||
# Return the same JSON types that a reader of analysis.json receives.
|
||||
analysis = json.loads(json.dumps(analysis))
|
||||
_write(directory / "analysis.json", analysis)
|
||||
(directory / "NOTES.md").write_text(_notes(analysis), encoding="utf-8")
|
||||
return analysis
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Measure proposals through the checkout runner before applying the acceptance rule."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .. import stats
|
||||
from . import accept, runner
|
||||
from .ablation import _findings, _write
|
||||
from .metric import search_outcome
|
||||
from .propose import Candidate
|
||||
from .table import FeatureTable
|
||||
|
||||
|
||||
def _fold_changes(pool_manifest: Path, table: FeatureTable, load_changes: Any) -> list[dict]:
|
||||
"""Resolve folds from table rows, rejecting conflicting change or repository identities."""
|
||||
from benchmarks.unseen.replay import load_slice
|
||||
|
||||
runner.refuse_pilot(pool_manifest, pilot_report=False)
|
||||
folds, repo_folds = {}, {}
|
||||
for row in table.rows:
|
||||
key, repo, fold = row["change_id"], row["repo"], row.get("fold")
|
||||
if type(fold) is not int or fold < 0:
|
||||
raise ValueError("each table row needs a non-negative fold")
|
||||
if key in folds and folds[key] != (repo, fold):
|
||||
raise ValueError("rows of a change must share repository and fold")
|
||||
if repo in repo_folds and repo_folds[repo] != fold:
|
||||
raise ValueError("a repository cannot cross folds")
|
||||
folds[key], repo_folds[repo] = (repo, fold), fold
|
||||
loaded = (load_changes or load_slice)(pool_manifest, "1")
|
||||
if len({c["id"] for c in loaded}) != len(loaded):
|
||||
raise ValueError("change IDs must be unique")
|
||||
selected = []
|
||||
for change in loaded:
|
||||
if change["id"] not in folds:
|
||||
raise ValueError("pool changes must exist in the feature table")
|
||||
repo, fold = folds[change["id"]]
|
||||
if change["repo"] != repo:
|
||||
raise ValueError("pool and table disagree on repository")
|
||||
selected.append({**change, "fold": fold})
|
||||
return selected
|
||||
|
||||
|
||||
def score_folds(
|
||||
program: Any,
|
||||
*,
|
||||
program_id: str,
|
||||
mask: Sequence[str],
|
||||
pool_manifest: Path,
|
||||
repos: Mapping[str, str],
|
||||
table: FeatureTable,
|
||||
folds: Sequence[int],
|
||||
out_dir: Path,
|
||||
lm: Any,
|
||||
budget: Mapping[str, float],
|
||||
repeats: int = 1,
|
||||
load_changes: Any = None,
|
||||
tools_for: Any = None,
|
||||
) -> Path:
|
||||
"""Run selected folds in repeat directories numbered from zero, with runner resumability.
|
||||
|
||||
The table supplies fold membership; the runner computes inputs at each change's head and
|
||||
binds repository tools there. The saved study fixes the sample before any records are written.
|
||||
Budget applies to each runner pass, as it does for ablations.
|
||||
"""
|
||||
import dspy
|
||||
|
||||
from .program import repository_tools
|
||||
|
||||
if type(repeats) is not int or repeats < 1:
|
||||
raise ValueError("repeats must be a positive integer")
|
||||
if not folds or any(type(f) is not int or f < 0 for f in folds) or len(set(folds)) != len(folds):
|
||||
raise ValueError("folds must be distinct non-negative integers")
|
||||
pool_manifest, out = Path(pool_manifest), Path(out_dir)
|
||||
changes = [c for c in _fold_changes(pool_manifest, table, load_changes) if c["fold"] in folds]
|
||||
if not changes:
|
||||
raise ValueError("selected folds contain no changes")
|
||||
study = dict(
|
||||
changes=changes,
|
||||
repeats=repeats,
|
||||
program_id=program_id,
|
||||
mask=sorted(set(mask)),
|
||||
folds=sorted(folds),
|
||||
table_digest=table.digest(),
|
||||
pool_digest=hashlib.sha256(pool_manifest.read_bytes()).hexdigest(),
|
||||
)
|
||||
path = out / "study.json"
|
||||
if path.exists() and json.loads(path.read_text()) != study:
|
||||
raise ValueError("study differs from the saved fold run")
|
||||
if not path.exists() and out.exists() and any(out.iterdir()):
|
||||
raise ValueError("cannot adopt records without a saved fold study")
|
||||
_write(path, study)
|
||||
for repeat in range(repeats):
|
||||
with dspy.context(lm=lm, adapter=dspy.ChatAdapter()):
|
||||
runner.run(
|
||||
pool_manifest,
|
||||
repos,
|
||||
out / str(repeat),
|
||||
program=program,
|
||||
program_id=program_id,
|
||||
mask=mask,
|
||||
only="all",
|
||||
pilot_report=False,
|
||||
budget=budget,
|
||||
load_changes=lambda _path, _slice: changes,
|
||||
tools_for=tools_for or repository_tools,
|
||||
versions={"dspy": dspy.__version__},
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _outcome(record: Mapping, change: Mapping, tolerance: int) -> dict:
|
||||
"""Keep failed, unparsed and absent records as misses, exactly as ablation analysis does."""
|
||||
failed = record.get("status") == "instrument_failure"
|
||||
parsed = bool(record.get("parse_failed"))
|
||||
defects = record.get("push", {}).get("admission", {}).get("defects", [])
|
||||
if not record.get("done") or failed or parsed:
|
||||
defects = []
|
||||
return dict(
|
||||
defects=defects,
|
||||
hit=search_outcome(change, _findings(defects), tolerance=tolerance).hit,
|
||||
instrument_failures=int(failed),
|
||||
parse_failures=int(parsed),
|
||||
missing_records=int(not record),
|
||||
)
|
||||
|
||||
|
||||
def _estimate(rows: list[dict], *, seed: int, resamples: int, stat: Any = None) -> stats.Estimate:
|
||||
"""Compare candidate minus current while resampling whole repositories."""
|
||||
import pandas as pd
|
||||
|
||||
return stats.paired_bootstrap(
|
||||
pd.DataFrame(rows, columns=["repo", "a", "b"]), "repo", "a", "b", seed=seed, resamples=resamples, stat=stat
|
||||
)
|
||||
|
||||
|
||||
def _families(units: list[dict], arms: Sequence[str]) -> list[str]:
|
||||
"""Include labelled families and families reported only on ordinary pushes."""
|
||||
return sorted(
|
||||
{u["change"]["family"] for u in units if u["change"].get("family")}
|
||||
| {d["family"] for u in units for arm in arms for d in u[arm]["defects"] if d.get("family")}
|
||||
)
|
||||
|
||||
|
||||
def _paired(units: list[dict], family: str | None, *, seed: int, resamples: int) -> tuple[stats.Estimate, stats.Estimate]:
|
||||
"""Compare hits on labelled positives and family suspicions over every ordinary push."""
|
||||
positives = [u for u in units if u["change"]["kind"] == "vulnerability" and (family is None or u["change"].get("family") == family)]
|
||||
ordinary = [u for u in units if u["change"]["kind"] == "ordinary"]
|
||||
recall = _estimate(
|
||||
[dict(repo=u["change"]["repo"], a=int(u["a"]["hit"]), b=int(u["b"]["hit"])) for u in positives], seed=seed, resamples=resamples
|
||||
)
|
||||
suspicions = _estimate(
|
||||
[
|
||||
{
|
||||
"repo": u["change"]["repo"],
|
||||
**{arm: sum(family is None or d.get("family") == family for d in u[arm]["defects"]) for arm in ("a", "b")},
|
||||
}
|
||||
for u in ordinary
|
||||
],
|
||||
seed=seed,
|
||||
resamples=resamples,
|
||||
)
|
||||
return recall, suspicions
|
||||
|
||||
|
||||
def fold_results(
|
||||
current_dir: Path, candidate_dir: Path, changes: Sequence[Mapping], *, scope: str, tolerance: int, seed: int, resamples: int = 2000
|
||||
) -> dict[str, accept.FoldResult]:
|
||||
"""Return acceptance inputs on (change, repeat) pairs, including missing records as misses.
|
||||
|
||||
Pass the changes for this scope. Saved studies fix the expected repeats; for standalone record
|
||||
fixtures, numeric child directories supply repeats. The union keeps an entirely missing arm.
|
||||
"""
|
||||
if scope not in ("compile", "holdout"):
|
||||
raise ValueError("scope must be compile or holdout")
|
||||
if tolerance < 0 or resamples < 2:
|
||||
raise ValueError("tolerance must be non-negative and resamples at least two")
|
||||
changes = list(changes)
|
||||
if len({c["id"] for c in changes}) != len(changes):
|
||||
raise ValueError("change IDs must be unique")
|
||||
directories = {"a": Path(current_dir), "b": Path(candidate_dir)}
|
||||
studies, repeats = {}, set()
|
||||
for arm, directory in directories.items():
|
||||
path = directory / "study.json"
|
||||
if path.exists():
|
||||
study = studies[arm] = json.loads(path.read_text())
|
||||
saved = {c["id"]: c for c in study["changes"]}
|
||||
if any(c["id"] not in saved or {"fold": saved[c["id"]]["fold"], **c} != saved[c["id"]] for c in changes):
|
||||
raise ValueError("analysis changes differ from the saved study")
|
||||
repeats.update(range(study["repeats"]))
|
||||
elif directory.exists():
|
||||
repeats.update(int(p.name) for p in directory.iterdir() if p.is_dir() and p.name.isdecimal())
|
||||
if len(studies) == 2 and studies["a"]["repeats"] != studies["b"]["repeats"]:
|
||||
raise ValueError("paired studies must have the same repeats")
|
||||
units = []
|
||||
for change in changes:
|
||||
for repeat in sorted(repeats or {0}):
|
||||
unit = dict(change=change, repeat=repeat)
|
||||
for arm, directory in directories.items():
|
||||
path = runner.record_path(directory / str(repeat), change["id"])
|
||||
record = json.loads(path.read_text()) if path.exists() else {}
|
||||
if record and arm in studies:
|
||||
study = studies[arm]
|
||||
saved_change = next(c for c in study["changes"] if c["id"] == change["id"])
|
||||
if (
|
||||
record.get("id") != change["id"]
|
||||
or record.get("program_id") != study["program_id"]
|
||||
or set(record.get("mask", [])) != set(study["mask"])
|
||||
or record.get("input_digest") != runner.input_digest(saved_change, study["program_id"], study["mask"])
|
||||
):
|
||||
raise ValueError("record does not match the saved change, program or mask")
|
||||
unit[arm] = _outcome(record, change, tolerance)
|
||||
units.append(unit)
|
||||
return {
|
||||
family: accept.FoldResult(scope, family, *_paired(units, None if family == "pooled" else family, seed=seed, resamples=resamples))
|
||||
for family in ["pooled", *_families(units, ("a", "b"))]
|
||||
}
|
||||
|
||||
|
||||
def _candidate_program(candidate: Candidate, tools_for: Any = None) -> Any:
|
||||
"""Restore either a one-call program or its tool loop without losing the proposed state."""
|
||||
from .program import HuntProgram, program_id, repository_tools
|
||||
|
||||
uses_tools = "hunt.react" in candidate.state
|
||||
tools = (tools_for or repository_tools)(Path.cwd()) if uses_tools else ()
|
||||
program = HuntProgram(mask=candidate.mask, tools=tools)
|
||||
if set(program.dump_state()) != set(candidate.state):
|
||||
raise ValueError("unsupported candidate program structure")
|
||||
program.load_state(candidate.state)
|
||||
if program_id(program, mask=candidate.mask) != candidate.program_id:
|
||||
raise ValueError("candidate program ID does not match its saved state")
|
||||
return program
|
||||
|
||||
|
||||
def round_trip(
|
||||
current: Any,
|
||||
candidate: Candidate,
|
||||
*,
|
||||
current_id: str,
|
||||
pool_manifest: Path,
|
||||
repos: Mapping[str, str],
|
||||
table: FeatureTable,
|
||||
compile_folds: Sequence[int],
|
||||
holdout_folds: Sequence[int],
|
||||
out_dir: Path,
|
||||
lm: Any,
|
||||
budget: Mapping[str, float],
|
||||
rule: accept.Rule,
|
||||
repeats: int = 1,
|
||||
tolerance: int = 10,
|
||||
seed: int = 0,
|
||||
resamples: int = 2000,
|
||||
load_changes: Any = None,
|
||||
tools_for: Any = None,
|
||||
) -> accept.Decision:
|
||||
"""Score both programs on both scopes, decide, and retain compile-only rejection history."""
|
||||
from .program import program_id
|
||||
|
||||
if not compile_folds or not holdout_folds or set(compile_folds) & set(holdout_folds):
|
||||
raise ValueError("compile and holdout folds must be nonempty and disjoint")
|
||||
if current_id != program_id(current, mask=candidate.base_mask) or tuple(current.mask) != candidate.base_mask:
|
||||
raise ValueError("current program does not match its ID and base mask")
|
||||
candidate_program = _candidate_program(candidate, tools_for)
|
||||
changes = _fold_changes(Path(pool_manifest), table, load_changes)
|
||||
results = {}
|
||||
for scope, folds in (("compile", compile_folds), ("holdout", holdout_folds)):
|
||||
paths = []
|
||||
for arm, program, pid, mask in (
|
||||
("current", current, current_id, candidate.base_mask),
|
||||
("candidate", candidate_program, candidate.program_id, candidate.mask),
|
||||
):
|
||||
paths.append(
|
||||
score_folds(
|
||||
program,
|
||||
program_id=pid,
|
||||
mask=mask,
|
||||
pool_manifest=pool_manifest,
|
||||
repos=repos,
|
||||
table=table,
|
||||
folds=folds,
|
||||
out_dir=Path(out_dir) / scope / arm,
|
||||
lm=lm,
|
||||
budget=budget,
|
||||
repeats=repeats,
|
||||
load_changes=load_changes,
|
||||
tools_for=tools_for,
|
||||
)
|
||||
)
|
||||
results[scope] = fold_results(
|
||||
*paths, [c for c in changes if c["fold"] in folds], scope=scope, tolerance=tolerance, seed=seed, resamples=resamples
|
||||
)
|
||||
decision = accept.decide(current_id, candidate, results["compile"], results["holdout"], rule)
|
||||
if not decision.accepted:
|
||||
accept.record_rejection(candidate.out_dir, candidate, decision)
|
||||
return decision
|
||||
Reference in New Issue
Block a user