mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
473 lines
23 KiB
Python
473 lines
23 KiB
Python
|
"""Report one accepted program on the held-out pilot, with instrument proof before scores."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import fcntl
|
||
|
|
import json
|
||
|
|
import math
|
||
|
|
import os
|
||
|
|
from collections.abc import Mapping, Sequence
|
||
|
|
from copy import deepcopy
|
||
|
|
from dataclasses import asdict
|
||
|
|
from datetime import UTC, datetime
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
from .. import stats
|
||
|
|
from . import accept, contamination, runner
|
||
|
|
from .ablation import _arm_summary, _findings, _write
|
||
|
|
from .metric import search_outcome
|
||
|
|
from .propose import Candidate
|
||
|
|
from .rounds import _candidate_program, _estimate, _families, _outcome, _paired
|
||
|
|
|
||
|
|
|
||
|
|
def _reserve(root: Path, program_id: str) -> Path:
|
||
|
|
"""Reserve a program under a file lock and flush the ledger before starting its run."""
|
||
|
|
date = datetime.now(UTC).date().isoformat()
|
||
|
|
if not program_id or any(c not in "0123456789abcdef" for c in program_id):
|
||
|
|
raise ValueError("program ID must be a hexadecimal content address")
|
||
|
|
root.mkdir(parents=True, exist_ok=True)
|
||
|
|
directory = root / f"{date}-hunt-gain-{program_id[:12]}"
|
||
|
|
with (root / "hunt-pilot-runs.jsonl").open("a+", encoding="utf-8") as stream:
|
||
|
|
fcntl.flock(stream, fcntl.LOCK_EX)
|
||
|
|
stream.seek(0)
|
||
|
|
if any(json.loads(line)["program_id"] == program_id for line in stream if line.strip()):
|
||
|
|
raise ValueError("this program ID already has a pilot run")
|
||
|
|
directory.mkdir(exist_ok=False)
|
||
|
|
stream.write(json.dumps(dict(program_id=program_id, date=date, measurement_directory=str(directory.resolve()))) + "\n")
|
||
|
|
stream.flush()
|
||
|
|
os.fsync(stream.fileno())
|
||
|
|
return directory
|
||
|
|
|
||
|
|
|
||
|
|
def _records(directory: Path, changes: Sequence[Mapping]) -> list[dict]:
|
||
|
|
"""Load benchmark records and retain every expected change, even after a budget stop."""
|
||
|
|
from benchmarks.unseen import score
|
||
|
|
|
||
|
|
records = score.load_records(Path(directory)) if (Path(directory) / "inputs.json").exists() else []
|
||
|
|
indexed = {r["id"]: r for r in records}
|
||
|
|
if len(indexed) != len(records) or set(indexed) - {c["id"] for c in changes}:
|
||
|
|
raise ValueError("records do not match the pilot population")
|
||
|
|
result = []
|
||
|
|
for change in changes:
|
||
|
|
record = indexed.get(change["id"], dict(change))
|
||
|
|
if any(record.get(key) != change.get(key) for key in ("repo", "base", "head", "kind", "family", "files", "targets")):
|
||
|
|
raise ValueError("record describes a different pilot change")
|
||
|
|
result.append(record)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _instrument(records: Sequence[Mapping]) -> dict:
|
||
|
|
"""Record changed-file bytes and verified revisions; unreadable and absent changes fail."""
|
||
|
|
from benchmarks.unseen import score
|
||
|
|
|
||
|
|
proof = []
|
||
|
|
for record in records:
|
||
|
|
instrument = record.get("instrument", {})
|
||
|
|
sizes = instrument.get("changed_bytes", {})
|
||
|
|
failed = not record.get("done") or record.get("status") == "instrument_failure" or score.coverage(record) == "instrument_failure"
|
||
|
|
proof.append(
|
||
|
|
dict(
|
||
|
|
id=record["id"],
|
||
|
|
repo=record["repo"],
|
||
|
|
base=record["base"],
|
||
|
|
head=record["head"],
|
||
|
|
base_verified=bool(instrument.get("base_verified")),
|
||
|
|
head_verified=bool(instrument.get("head_verified")),
|
||
|
|
changed_bytes={f["path"]: sizes.get(f["path"], 0) for f in record.get("files", [])},
|
||
|
|
instrument=dict(instrument),
|
||
|
|
instrument_failure=failed,
|
||
|
|
reason=instrument.get("failure") or ("missing or unreadable record" if failed else None),
|
||
|
|
)
|
||
|
|
)
|
||
|
|
failures = sum(row["instrument_failure"] for row in proof)
|
||
|
|
return dict(
|
||
|
|
changes=len(proof),
|
||
|
|
failures=failures,
|
||
|
|
failure_fraction=failures / len(proof) if proof else None,
|
||
|
|
valid=bool(proof) and failures / len(proof) <= 0.05,
|
||
|
|
records=proof,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _clean(records: Sequence[Mapping]) -> list[dict]:
|
||
|
|
"""Remove stale findings on failures while preserving the scorer's coverage checks."""
|
||
|
|
from benchmarks.unseen import score
|
||
|
|
|
||
|
|
result = []
|
||
|
|
for original in records:
|
||
|
|
record = deepcopy(original)
|
||
|
|
if not record.get("done") or record.get("status") == "instrument_failure" or score.coverage(record) == "instrument_failure":
|
||
|
|
record["status"] = "instrument_failure"
|
||
|
|
record.setdefault("instrument", {})["failure"] = "instrument_failure"
|
||
|
|
outcome = _outcome(record, record, 10)
|
||
|
|
if not record.get("done") or record.get("status") == "instrument_failure" or record.get("parse_failed"):
|
||
|
|
record["push"] = {"admission": {"defects": []}, "quick_tier": []}
|
||
|
|
else:
|
||
|
|
record.setdefault("push", {}).setdefault("admission", {})["defects"] = outcome["defects"]
|
||
|
|
result.append(record)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _rate(values: list[float], repos: list[str], *, binary: bool = False, scale: float = 1) -> dict:
|
||
|
|
"""Report a mean with its count and a repository-clustered interval, plus Wilson for proportions."""
|
||
|
|
from benchmarks.unseen import score
|
||
|
|
|
||
|
|
interval = score.cluster_interval(list(zip(repos, values, strict=True)), seed=0)
|
||
|
|
result = dict(
|
||
|
|
rate=scale * sum(values) / len(values) if values else None,
|
||
|
|
count=sum(values),
|
||
|
|
n=len(values),
|
||
|
|
groups=len(set(repos)),
|
||
|
|
ci=[scale * v for v in interval] if values else None,
|
||
|
|
)
|
||
|
|
if binary:
|
||
|
|
wilson = stats.wilson(int(sum(values)), len(values))
|
||
|
|
result["wilson95"] = [scale * v for v in wilson] if wilson else None
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _family_records(records: Sequence[Mapping], family: str | None) -> list[dict]:
|
||
|
|
"""Keep a family's positives and count only its findings over all ordinary pushes."""
|
||
|
|
result = []
|
||
|
|
for original in records:
|
||
|
|
if family is not None and original["kind"] == "vulnerability" and original.get("family") != family:
|
||
|
|
continue
|
||
|
|
record = deepcopy(original)
|
||
|
|
admission = record.setdefault("push", {}).setdefault("admission", {})
|
||
|
|
admission["defects"] = [d for d in admission.get("defects", []) if family is None or d.get("family") == family]
|
||
|
|
result.append(record)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _summaries(records: list[dict], families: list[str]) -> dict:
|
||
|
|
"""Use the pilot functions for push metrics and the hunt metric for search outcomes."""
|
||
|
|
from benchmarks.unseen import score
|
||
|
|
from benchmarks.valid import metrics
|
||
|
|
|
||
|
|
summaries = {}
|
||
|
|
for family in ["pooled", *families]:
|
||
|
|
part = _family_records(records, None if family == "pooled" else family)
|
||
|
|
positives = [r for r in part if r["kind"] == "vulnerability"]
|
||
|
|
ordinary = [r for r in part if r["kind"] == "ordinary"]
|
||
|
|
outcomes = {r["id"]: search_outcome(r, _findings(r.get("push", {}).get("admission", {}).get("defects", []))) for r in part}
|
||
|
|
search = dict(
|
||
|
|
recall=_rate([int(outcomes[r["id"]].hit) for r in positives], [r["repo"] for r in positives], binary=True),
|
||
|
|
suspicions_per_push=_rate([outcomes[r["id"]].suspicions for r in ordinary], [r["repo"] for r in ordinary]),
|
||
|
|
)
|
||
|
|
# The whole run already passed its instrument gate. The pilot scorer excludes these failures
|
||
|
|
# from push denominators, while the search summaries above keep them as misses.
|
||
|
|
usable = [r for r in part if score.coverage(r) != "instrument_failure"]
|
||
|
|
if usable:
|
||
|
|
scored = score.score(metrics.scoring_records(usable))
|
||
|
|
push = metrics.summarize(usable, scored=scored)
|
||
|
|
else:
|
||
|
|
push = {key: score.rate([]) for key in ("recall", "precision", "false_alarms_per_100_pushes")}
|
||
|
|
summaries[family] = dict(search=search, push=push)
|
||
|
|
return summaries
|
||
|
|
|
||
|
|
|
||
|
|
def _curve(records: list[dict]) -> tuple[list[dict], dict]:
|
||
|
|
"""Extend ablation confidence cuts with false alarms, intervals and the two gate bounds."""
|
||
|
|
from benchmarks.unseen import score
|
||
|
|
|
||
|
|
units = [dict(change=r, after=_outcome(r, r, 10)) for r in records]
|
||
|
|
curve = _arm_summary(units, "after", 10)["curve"]
|
||
|
|
for point in curve:
|
||
|
|
cut = point["confidence"]
|
||
|
|
filtered = deepcopy(records)
|
||
|
|
for record in filtered:
|
||
|
|
defects = record.get("push", {}).get("admission", {}).get("defects", [])
|
||
|
|
record.setdefault("push", {}).setdefault("admission", {})["defects"] = [
|
||
|
|
d
|
||
|
|
for d in defects
|
||
|
|
if isinstance(d.get("confidence"), int | float) and math.isfinite(d["confidence"]) and d["confidence"] >= cut
|
||
|
|
]
|
||
|
|
positives = [r for r in filtered if r["kind"] == "vulnerability"]
|
||
|
|
ordinary = [r for r in filtered if r["kind"] == "ordinary"]
|
||
|
|
hits = [int(search_outcome(r, _findings(r["push"]["admission"]["defects"])).hit) for r in positives]
|
||
|
|
point["estimates"] = dict(
|
||
|
|
recall=_rate(hits, [r["repo"] for r in positives], binary=True),
|
||
|
|
suspicions_per_push=_rate([len(r["push"]["admission"]["defects"]) for r in ordinary], [r["repo"] for r in ordinary]),
|
||
|
|
false_alarms_per_100_pushes=_rate(
|
||
|
|
[int(score.false_alarm(r)) for r in ordinary], [r["repo"] for r in ordinary], binary=True, scale=100
|
||
|
|
),
|
||
|
|
)
|
||
|
|
point["false_alarms_per_100_pushes"] = point["estimates"]["false_alarms_per_100_pushes"]["rate"]
|
||
|
|
high_recall = [p for p in curve if p["recall"] is not None and p["recall"] >= 0.9 and p["false_alarms_per_100_pushes"] is not None]
|
||
|
|
low_alarm = [
|
||
|
|
p
|
||
|
|
for p in curve
|
||
|
|
if p["false_alarms_per_100_pushes"] is not None and p["false_alarms_per_100_pushes"] < 10 and p["recall"] is not None
|
||
|
|
]
|
||
|
|
return curve, dict(
|
||
|
|
false_alarms_at_recall_90=min(high_recall, key=lambda p: p["false_alarms_per_100_pushes"]) if high_recall else None,
|
||
|
|
recall_under_10_false_alarms=max(low_alarm, key=lambda p: p["recall"]) if low_alarm else None,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _precision(column: Any) -> float | None:
|
||
|
|
"""Reduce paired (correctly flagged, flagged) counts to push precision."""
|
||
|
|
flagged = sum(value[1] for value in column)
|
||
|
|
return sum(value[0] for value in column) / flagged if flagged else None
|
||
|
|
|
||
|
|
|
||
|
|
def _baseline_comparison(baselines: list[list[dict]], records: list[dict], families: list[str]) -> dict:
|
||
|
|
"""Pair every baseline repeat with the one accepted run, without inventing new repetitions."""
|
||
|
|
from benchmarks.unseen import score
|
||
|
|
from benchmarks.valid import metrics
|
||
|
|
|
||
|
|
units = [
|
||
|
|
dict(change=after, repeat=repeat, a=_outcome(before, before, 10), b=_outcome(after, after, 10))
|
||
|
|
for repeat, baseline in enumerate(baselines)
|
||
|
|
for before, after in zip(baseline, records, strict=True)
|
||
|
|
]
|
||
|
|
result = {}
|
||
|
|
for name in ["pooled", *families]:
|
||
|
|
family = None if name == "pooled" else name
|
||
|
|
recall, suspicions = _paired(units, family, seed=0, resamples=stats.RESAMPLES)
|
||
|
|
positives = [u for u in units if u["change"]["kind"] == "vulnerability" and (family is None or u["change"].get("family") == family)]
|
||
|
|
paired = dict(
|
||
|
|
search_recall=asdict(recall),
|
||
|
|
suspicions_per_push=asdict(suspicions),
|
||
|
|
mcnemar=asdict(stats.mcnemar_exact([u["a"]["hit"] for u in positives], [u["b"]["hit"] for u in positives])),
|
||
|
|
)
|
||
|
|
rows = {"recall": [], "false_alarms_per_100_pushes": [], "precision": []}
|
||
|
|
for baseline in baselines:
|
||
|
|
before = metrics.scoring_records(_family_records(baseline, family))
|
||
|
|
after = metrics.scoring_records(_family_records(records, family))
|
||
|
|
for a, b in zip(before, after, strict=True):
|
||
|
|
flags = [int(score.false_alarm(r)) for r in (a, b)]
|
||
|
|
hits = [int(bool(score.catches(r))) if r["kind"] == "vulnerability" else 0 for r in (a, b)]
|
||
|
|
key = "recall" if a["kind"] == "vulnerability" else "false_alarms_per_100_pushes"
|
||
|
|
values = hits if key == "recall" else [100 * v for v in flags]
|
||
|
|
rows[key].append(dict(repo=a["repo"], a=values[0], b=values[1]))
|
||
|
|
rows["precision"].append(dict(repo=a["repo"], a=(hits[0], flags[0]), b=(hits[1], flags[1])))
|
||
|
|
for metric, frame in rows.items():
|
||
|
|
paired[metric] = asdict(_estimate(frame, seed=0, resamples=stats.RESAMPLES, stat=_precision if metric == "precision" else None))
|
||
|
|
for metric in ("recall", "false_alarms_per_100_pushes"):
|
||
|
|
paired[metric + "_mcnemar"] = asdict(
|
||
|
|
stats.mcnemar_exact([bool(r["a"]) for r in rows[metric]], [bool(r["b"]) for r in rows[metric]])
|
||
|
|
)
|
||
|
|
result[name] = paired
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _before(pilot_metrics: Mapping[str, Path]) -> dict:
|
||
|
|
"""Copy the two recorded metric files unchanged, with their source paths."""
|
||
|
|
if set(pilot_metrics) != {"raw", "filtered"}:
|
||
|
|
raise ValueError("pilot_metrics must map raw and filtered to their recorded JSON files")
|
||
|
|
return {name: dict(source=str(Path(path)), metrics=json.loads(Path(path).read_text())) for name, path in pilot_metrics.items()}
|
||
|
|
|
||
|
|
|
||
|
|
def _format(metric: Mapping | None) -> str:
|
||
|
|
"""Show the estimate, interval and count together, leaving missing measurements explicit."""
|
||
|
|
if not metric or metric.get("rate") is None:
|
||
|
|
return "not recorded" if not metric else f"not measured (n={metric.get('n', 0)})"
|
||
|
|
intervals = []
|
||
|
|
for key, label in (("ci", "cluster 95%"), ("cluster95", "cluster 95%"), ("wilson95", "Wilson 95%")):
|
||
|
|
ci = metric.get(key)
|
||
|
|
if ci and all(v is not None for v in ci):
|
||
|
|
intervals.append(f"{label} {ci[0]:.6g} to {ci[1]:.6g}")
|
||
|
|
return f"{metric['rate']:.6g}; {'; '.join(intervals)}; count={metric['count']}, n={metric['n']}"
|
||
|
|
|
||
|
|
|
||
|
|
def _notes(analysis: dict) -> str:
|
||
|
|
"""Write readable before-and-after tables and the evidence needed to interpret them."""
|
||
|
|
lines = ["# Hunter pilot gain", "", f"Program: `{analysis['program_id']}`", ""]
|
||
|
|
proof = analysis["instrument"]
|
||
|
|
lines += [
|
||
|
|
"## Instrument",
|
||
|
|
"",
|
||
|
|
f"{proof['failures']} instrument failures across {proof['changes']} changes.",
|
||
|
|
"Bytes per changed file and verified base and head are in instrument.json.",
|
||
|
|
"",
|
||
|
|
]
|
||
|
|
if not analysis["valid"]:
|
||
|
|
lines += [f"Run invalid: {analysis['reason']}. No performance numbers are quoted.", ""]
|
||
|
|
else:
|
||
|
|
lines += [
|
||
|
|
"## Before and after",
|
||
|
|
"",
|
||
|
|
"Before values are quoted as recorded; unavailable family or search values are not inferred.",
|
||
|
|
"Recall and precision are proportions. False alarms are per 100 ordinary pushes.",
|
||
|
|
"Push metrics exclude instrument failures as in the pilot. Search metrics and paired comparisons retain failures as misses.",
|
||
|
|
"The confidence curve describes the search output; it does not choose a threshold or change the classifier.",
|
||
|
|
"",
|
||
|
|
"| Family | Metric | Before raw | Before filtered | After |",
|
||
|
|
"|---|---|---|---|---|",
|
||
|
|
]
|
||
|
|
for family, summary in analysis["after"].items():
|
||
|
|
for stage, key, label in (
|
||
|
|
("search", "recall", "Search recall"),
|
||
|
|
("search", "suspicions_per_push", "Suspicions per ordinary push"),
|
||
|
|
("push", "recall", "Push recall"),
|
||
|
|
("push", "false_alarms_per_100_pushes", "False alarms per 100"),
|
||
|
|
("push", "precision", "Precision"),
|
||
|
|
):
|
||
|
|
before = [
|
||
|
|
analysis["before"][name]["metrics"].get(key) if family == "pooled" and stage == "push" else None
|
||
|
|
for name in ("raw", "filtered")
|
||
|
|
]
|
||
|
|
lines.append(f"| {family} | {label} | {_format(before[0])} | {_format(before[1])} | {_format(summary[stage][key])} |")
|
||
|
|
for name, recorded in analysis["before"].items():
|
||
|
|
lines += ["", f"Recorded {name} source: `{recorded['source']}`."]
|
||
|
|
if analysis.get("baseline"):
|
||
|
|
lines += [
|
||
|
|
"",
|
||
|
|
"## Baseline repeats",
|
||
|
|
"",
|
||
|
|
"Each repeat is paired with the single accepted run; intervals resample repositories.",
|
||
|
|
"",
|
||
|
|
"| Repeat | Family | Metric | Baseline | After |",
|
||
|
|
"|---|---|---|---|---|",
|
||
|
|
]
|
||
|
|
for repeat, baseline in enumerate(analysis["baseline"]):
|
||
|
|
if not baseline["valid"]:
|
||
|
|
lines += [f"Baseline repeat {repeat} is invalid; its performance numbers and paired gain are not quoted."]
|
||
|
|
continue
|
||
|
|
for family, summary in baseline["summary"].items():
|
||
|
|
for stage, keys in (
|
||
|
|
("search", ("recall", "suspicions_per_push")),
|
||
|
|
("push", ("recall", "false_alarms_per_100_pushes", "precision")),
|
||
|
|
):
|
||
|
|
for key in keys:
|
||
|
|
lines.append(
|
||
|
|
f"| {repeat} | {family} | {stage} {key} | {_format(summary[stage][key])} | "
|
||
|
|
f"{_format(analysis['after'][family][stage][key])} |"
|
||
|
|
)
|
||
|
|
if "paired" in analysis:
|
||
|
|
lines += [
|
||
|
|
"",
|
||
|
|
"Paired differences and exact McNemar results:",
|
||
|
|
"",
|
||
|
|
"```json",
|
||
|
|
json.dumps(analysis["paired"], indent=2),
|
||
|
|
"```",
|
||
|
|
]
|
||
|
|
lines += ["", "## Workload", ""]
|
||
|
|
for name, value in analysis["workload"].items():
|
||
|
|
lines.append(f"{name.replace('_', ' ')}: {_format(value)}.")
|
||
|
|
lines += ["", "## Gate distance", ""]
|
||
|
|
for name, point in analysis["gate_distance"].items():
|
||
|
|
label = (
|
||
|
|
"Lowest false alarms at recall at least 0.9"
|
||
|
|
if name == "false_alarms_at_recall_90"
|
||
|
|
else "Highest recall under 10 false alarms per 100"
|
||
|
|
)
|
||
|
|
lines.append(
|
||
|
|
f"{label}: "
|
||
|
|
+ (
|
||
|
|
"the curve never reaches this gate bound."
|
||
|
|
if point is None
|
||
|
|
else f"recall {_format(point['estimates']['recall'])}; "
|
||
|
|
f"false alarms {_format(point['estimates']['false_alarms_per_100_pushes'])}."
|
||
|
|
)
|
||
|
|
)
|
||
|
|
lines += ["", "## Acceptance comparison", "", "```json", json.dumps(analysis["comparison"], indent=2), "```", ""]
|
||
|
|
return "\n".join(lines)
|
||
|
|
|
||
|
|
|
||
|
|
def pilot_gain(
|
||
|
|
candidate: Candidate,
|
||
|
|
decision: accept.Decision,
|
||
|
|
*,
|
||
|
|
pilot_manifest: Path,
|
||
|
|
repos: Mapping[str, str],
|
||
|
|
training_runs: Sequence[Path],
|
||
|
|
out_root: Path,
|
||
|
|
lm: Any,
|
||
|
|
budget: Mapping[str, float],
|
||
|
|
baseline_runs: Sequence[Path] = (),
|
||
|
|
pilot_metrics: Mapping[str, Path],
|
||
|
|
tools_for: Any = None,
|
||
|
|
load_changes: Any = None,
|
||
|
|
) -> dict:
|
||
|
|
"""Run an accepted program once and write analysis.json, instrument.json and NOTES.md.
|
||
|
|
|
||
|
|
pilot_metrics maps raw and filtered to the original metric JSON files. Baseline runs are
|
||
|
|
single-repeat pilot record directories, each paired with this one accepted run. No command
|
||
|
|
line scorers or real model clients are constructed here. Budget exhaustion or more than five
|
||
|
|
percent instrument failures leaves an invalid measurement without performance numbers.
|
||
|
|
"""
|
||
|
|
import dspy
|
||
|
|
|
||
|
|
from benchmarks.unseen import score
|
||
|
|
from benchmarks.unseen.replay import load_slice
|
||
|
|
|
||
|
|
from .program import repository_tools
|
||
|
|
|
||
|
|
if not decision.accepted or decision.comparison.get("candidate_id") != candidate.program_id:
|
||
|
|
raise ValueError("pilot gain requires the accepted decision for this candidate")
|
||
|
|
training_runs = [Path(p) for p in training_runs]
|
||
|
|
if contamination.clean_runs(training_runs) != training_runs:
|
||
|
|
raise ValueError("contaminated training runs cannot report pilot gain")
|
||
|
|
before = _before(pilot_metrics)
|
||
|
|
program = _candidate_program(candidate, tools_for)
|
||
|
|
changes = list((load_changes or load_slice)(Path(pilot_manifest), "1"))
|
||
|
|
if not changes or len({c["id"] for c in changes}) != len(changes):
|
||
|
|
raise ValueError("pilot changes must be nonempty and have unique IDs")
|
||
|
|
directory = _reserve(Path(out_root), candidate.program_id)
|
||
|
|
with dspy.context(lm=lm, adapter=dspy.ChatAdapter()):
|
||
|
|
artifact = runner.run(
|
||
|
|
Path(pilot_manifest),
|
||
|
|
repos,
|
||
|
|
directory / "records",
|
||
|
|
program=program,
|
||
|
|
program_id=candidate.program_id,
|
||
|
|
mask=candidate.mask,
|
||
|
|
pilot_report=True,
|
||
|
|
only="all",
|
||
|
|
budget=budget,
|
||
|
|
tools_for=tools_for or repository_tools,
|
||
|
|
load_changes=lambda _path, _slice: changes,
|
||
|
|
versions={"dspy": dspy.__version__},
|
||
|
|
)
|
||
|
|
records = _records(directory / "records", changes)
|
||
|
|
proof = _instrument(records)
|
||
|
|
_write(directory / "instrument.json", proof)
|
||
|
|
analysis = dict(
|
||
|
|
program_id=candidate.program_id,
|
||
|
|
comparison=decision.comparison,
|
||
|
|
measurement_directory=directory.name,
|
||
|
|
valid=proof["valid"] and artifact is not None,
|
||
|
|
instrument={k: v for k, v in proof.items() if k != "records"},
|
||
|
|
)
|
||
|
|
if not analysis["valid"]:
|
||
|
|
analysis["reason"] = "more than 5% instrument failures" if not proof["valid"] else "runner budget exhausted"
|
||
|
|
else:
|
||
|
|
records = _clean(records)
|
||
|
|
# Run the same overall instrument gate and scorer as the pilot before quoting any scores.
|
||
|
|
score.score(records)
|
||
|
|
families = _families([dict(change=r, after=_outcome(r, r, 10)) for r in records], ("after",))
|
||
|
|
baseline_records = [_clean(_records(Path(path), changes)) for path in baseline_runs]
|
||
|
|
for baseline in baseline_records:
|
||
|
|
families = sorted(set(families) | set(_families([dict(change=r, before=_outcome(r, r, 10)) for r in baseline], ("before",))))
|
||
|
|
curve, distance = _curve(records)
|
||
|
|
counts = [search_outcome(r, _findings(r["push"]["admission"]["defects"])).suspicions for r in records]
|
||
|
|
analysis.update(
|
||
|
|
before=before,
|
||
|
|
after=_summaries(records, families),
|
||
|
|
curve=curve,
|
||
|
|
gate_distance=distance,
|
||
|
|
workload=dict(
|
||
|
|
suspicions_per_100_pushes=_rate(counts, [r["repo"] for r in records], scale=100),
|
||
|
|
flagged_pushes_per_100_pushes=_rate([int(n > 0) for n in counts], [r["repo"] for r in records], binary=True, scale=100),
|
||
|
|
),
|
||
|
|
)
|
||
|
|
if baseline_records:
|
||
|
|
analysis["baseline"] = []
|
||
|
|
for path, baseline in zip(baseline_runs, baseline_records, strict=True):
|
||
|
|
instrument = _instrument(baseline)
|
||
|
|
entry = dict(directory=str(path), valid=instrument["valid"], instrument=instrument)
|
||
|
|
if entry["valid"]:
|
||
|
|
entry["summary"] = _summaries(baseline, families)
|
||
|
|
analysis["baseline"].append(entry)
|
||
|
|
if all(row["valid"] for row in analysis["baseline"]):
|
||
|
|
analysis["paired"] = _baseline_comparison(baseline_records, records, families)
|
||
|
|
# Return the same JSON types that a reader of analysis.json receives.
|
||
|
|
analysis = json.loads(json.dumps(analysis))
|
||
|
|
_write(directory / "analysis.json", analysis)
|
||
|
|
(directory / "NOTES.md").write_text(_notes(analysis), encoding="utf-8")
|
||
|
|
return analysis
|