feat(hunt): candidates scored on folds through the runner; the pilot gain, once per accepted program (6.1)

rounds.py runs a candidate over compile or held-out folds through runner.run, so a tool-using
program has its tools bound to each change's checkout, and turns paired records into the
FoldResults accept.decide reads (closes the gap noted in bacd7a1). report.py runs an accepted
program once over the held-out pilot: a ledger line reserved before the run refuses a second run
per program id, contaminated or unchecked training runs are refused, the instrument is proved per
change and more than 5% failures makes the run invalid with no numbers quoted, the pilot's own
scorer and metrics give the after numbers, the recorded hunter numbers give the before, baseline
repeats pair with the single run, the proof arm's workload per 100 pushes is counted, and the gate
distance is read off the confidence curve. 124 hunt and stats tests pass. Implemented by Codex
(gpt-6-astra), gated here.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/test_hunt_report.py

@@ -0,0 +1,241 @@
+"""Pilot reports use small git fixtures and never run the held-out pilot."""
+
+import json
+from dataclasses import replace
+
+import pytest
+
+pytest.importorskip("dspy")
+pytest.importorskip("pandas")
+pytest.importorskip("statsmodels")
+
+from test_hunt_rounds import candidate_at, finding, model  # noqa: E402
+from test_hunt_rounds import population as population  # noqa: E402
+
+from benchmarks.unseen import score  # noqa: E402
+from benchmarks.valid import metrics  # noqa: E402
+from openultrasast.hunt import accept, report, runner  # noqa: E402
+
+
+@pytest.fixture
+def pilot_case(population, tmp_path):
+    """Create an accepted candidate, a checked training run and recorded before metrics."""
+    pool, repos, changes, _ = population
+    _, _, candidate = candidate_at(tmp_path / "proposal")
+    decision = accept.Decision(True, [], {"candidate_id": candidate.program_id, "accepted": True, "rule": {"version": "fixture"}})
+    training = tmp_path / "training"
+    training.mkdir()
+    (training / "contamination.json").write_text('{"contaminated": false}')
+    # These are fixture numbers in exactly the pilot metrics file's shape.
+    before = dict(
+        recall=score.rate([False, True]),
+        precision=score.rate([False, True, False]),
+        false_alarms_per_100_pushes={**score.rate([True, True]), "rate": 100, "wilson95": [34.237, 100]},
+    )
+    paths = {}
+    for name in ("raw", "filtered"):
+        path = tmp_path / f"metrics_b_{name}.json"
+        path.write_text(json.dumps(before))
+        paths[name] = path
+    kwargs = dict(
+        pilot_manifest=pool,
+        repos=repos,
+        training_runs=[training],
+        out_root=tmp_path / "reports",
+        lm=model([[finding()], [], [finding()], []]),
+        budget={"usd": 1, "calls": 100},
+        pilot_metrics=paths,
+        load_changes=lambda m, s: changes,
+    )
+    return candidate, decision, kwargs
+
+
+@pytest.mark.parametrize("accepted,matches", [(False, True), (True, False)])
+def test_refuse_unaccepted_or_unrelated_decision(pilot_case, accepted, matches):
+    """Only the decision for this accepted program authorizes a pilot attempt."""
+    candidate, decision, kwargs = pilot_case
+    decision = replace(decision, accepted=accepted, comparison={"candidate_id": candidate.program_id if matches else "other"})
+    with pytest.raises(ValueError, match="accepted"):
+        report.pilot_gain(candidate, decision, **kwargs)
+    assert not kwargs["out_root"].exists()
+
+
+@pytest.mark.parametrize("check", [None, {"contaminated": True}, {"contaminated": "false"}])
+def test_refuse_contaminated_or_unchecked_training(pilot_case, check):
+    """Missing, malformed and contaminated checks cannot lead to a pilot number."""
+    candidate, decision, kwargs = pilot_case
+    path = kwargs["training_runs"][0] / "contamination.json"
+    path.unlink() if check is None else path.write_text(json.dumps(check))
+    with pytest.raises(ValueError, match="unchecked|contaminat"):
+        report.pilot_gain(candidate, decision, **kwargs)
+    assert not kwargs["out_root"].exists()
+
+
+def test_valid_report_before_after_workload_curve_and_ledger(pilot_case, population):
+    """A readable run writes metrics, per-family tables, instrument proof and a durable reservation."""
+    candidate, decision, kwargs = pilot_case
+    analysis = report.pilot_gain(candidate, decision, **kwargs)
+    assert analysis["valid"]
+    directory = kwargs["out_root"] / analysis["measurement_directory"]
+    assert json.loads((directory / "analysis.json").read_text()) == analysis
+    instruments = json.loads((directory / "instrument.json").read_text())
+    assert instruments["failures"] == 0 and instruments["changes"] == 4
+    for row in instruments["records"]:
+        assert row["changed_bytes"]["app.py"] > 0
+        assert row["base_verified"] and row["head_verified"] and row["base"] and row["head"]
+    assert analysis["before"]["raw"]["metrics"] == json.loads(kwargs["pilot_metrics"]["raw"].read_text())
+    records = score.load_records(directory / "records")
+    assert analysis["after"]["pooled"]["push"] == metrics.summarize(records, scored=score.score(metrics.scoring_records(records)))
+    assert analysis["after"]["injection"]["search"]["recall"]["rate"] == 1
+    assert analysis["after"]["pooled"]["search"]["suspicions_per_push"]["rate"] == 0
+    assert analysis["workload"]["suspicions_per_100_pushes"]["rate"] == 50
+    assert analysis["workload"]["flagged_pushes_per_100_pushes"]["rate"] == 50
+    assert analysis["gate_distance"]["false_alarms_at_recall_90"]["false_alarms_per_100_pushes"] == 0
+    assert analysis["gate_distance"]["recall_under_10_false_alarms"]["recall"] == 1
+    notes = (directory / "NOTES.md").read_text()
+    for phrase in ("raw", "filtered", "injection", "pooled", "Workload", "Gate", "Instrument", candidate.program_id):
+        assert phrase in notes
+    ledger = [json.loads(line) for line in (kwargs["out_root"] / "hunt-pilot-runs.jsonl").read_text().splitlines()]
+    assert ledger[0]["program_id"] == candidate.program_id and ledger[0]["date"]
+    calls = len(kwargs["lm"].history)
+    with pytest.raises(ValueError, match="already"):
+        report.pilot_gain(candidate, decision, **kwargs)
+    assert len(kwargs["lm"].history) == calls
+
+
+def test_invalid_instrument_quotes_no_numbers(pilot_case, population):
+    """One unreadable change out of four invalidates the run before any metrics are computed."""
+    candidate, decision, kwargs = pilot_case
+    changes = [dict(c) for c in population[2]]
+    changes[0]["files"] = [{"path": "missing.py", "head_lines": [[1, 2]]}]
+    kwargs["load_changes"] = lambda m, s: changes
+    analysis = report.pilot_gain(candidate, decision, **kwargs)
+    assert not analysis["valid"] and analysis["instrument"]["failures"] == 1
+    for name in ("before", "after", "workload", "gate_distance", "paired"):
+        assert name not in analysis
+    directory = kwargs["out_root"] / analysis["measurement_directory"]
+    notes = (directory / "NOTES.md").read_text()
+    assert "invalid" in notes.lower() and "No performance numbers" in notes
+    assert "Wilson" not in notes and "0.5" not in notes
+    assert (directory / "instrument.json").is_file() and (directory / "analysis.json").is_file()
+
+
+def test_crash_consumes_program_id(pilot_case, monkeypatch):
+    """The reservation is on disk before runner.run starts, including a failed launch."""
+    candidate, decision, kwargs = pilot_case
+
+    def crash(*args, **kw):
+        assert kw["pilot_report"] is True and kw["only"] == "all"
+        ledger = kwargs["out_root"] / "hunt-pilot-runs.jsonl"
+        assert json.loads(ledger.read_text())["program_id"] == candidate.program_id
+        raise RuntimeError("fixture crash")
+
+    monkeypatch.setattr(runner, "run", crash)
+    with pytest.raises(RuntimeError, match="fixture crash"):
+        report.pilot_gain(candidate, decision, **kwargs)
+    with pytest.raises(ValueError, match="already"):
+        report.pilot_gain(candidate, decision, **kwargs)
+
+
+def test_baseline_repeats_are_paired_with_single_run(pilot_case, population, tmp_path):
+    """Two baseline repeats yield four positive pairs and two repository clusters."""
+    import dspy
+
+    candidate, decision, kwargs = pilot_case
+    base, _, _ = candidate_at(tmp_path / "baseline-program")
+    directories = []
+    for repeat in range(2):
+        out = tmp_path / f"baseline-{repeat}"
+        with dspy.context(lm=model([[]] * 4), adapter=dspy.ChatAdapter()):
+            runner.run(
+                population[0], population[1], out, program=base, program_id=candidate.base_program_id, load_changes=kwargs["load_changes"]
+            )
+        directories.append(out)
+    analysis = report.pilot_gain(candidate, decision, **{**kwargs, "baseline_runs": directories})
+    recall = analysis["paired"]["pooled"]["search_recall"]
+    assert recall["diff"] == 1 and recall["units"] == 4 and recall["groups"] == 2
+    assert recall["ci"] == [1, 1]
+    assert analysis["paired"]["pooled"]["mcnemar"]["c"] == 4
+    assert analysis["paired"]["pooled"]["recall_mcnemar"]["c"] == 4
+    assert analysis["paired"]["pooled"]["false_alarms_per_100_pushes"]["diff"] == 0
+    assert len(analysis["baseline"]) == 2
+
+
+def test_curve_reports_unreached_bounds_and_counts_unknown_families(pilot_case):
+    """Low recall and ordinary alarms leave both gate bounds explicitly unreached."""
+    candidate, decision, kwargs = pilot_case
+    kwargs["lm"] = model([[], [finding(0.4, "path"), finding(0.4)], [], [finding(0.4)]])
+    analysis = report.pilot_gain(candidate, decision, **kwargs)
+    assert analysis["gate_distance"]["false_alarms_at_recall_90"] is None
+    assert analysis["gate_distance"]["recall_under_10_false_alarms"] is None
+    assert analysis["after"]["path"]["search"]["recall"]["n"] == 0
+    assert analysis["after"]["path"]["search"]["suspicions_per_push"]["count"] == 1
+    point = analysis["curve"][0]
+    assert point["false_alarms_per_100_pushes"] == 100 and point["suspicions_per_push"] == 1.5
+    notes = (kwargs["out_root"] / analysis["measurement_directory"] / "NOTES.md").read_text()
+    assert "never reaches" in notes
+
+
+def test_gate_reads_both_bounds_from_different_confidence_cuts(pilot_case):
+    """A quiet high cut and a noisy complete cut report different gate distances."""
+    candidate, decision, kwargs = pilot_case
+    kwargs["lm"] = model([[finding(0.9)], [finding(0.4)], [finding(0.4)], [finding(0.4)]])
+    analysis = report.pilot_gain(candidate, decision, **kwargs)
+    assert [p["confidence"] for p in analysis["curve"]] == [0.9, 0.4]
+    assert analysis["gate_distance"]["false_alarms_at_recall_90"]["false_alarms_per_100_pushes"] == 100
+    assert analysis["gate_distance"]["recall_under_10_false_alarms"]["recall"] == 0.5
+    assert all(p["estimates"]["recall"]["n"] == 2 and p["estimates"]["recall"]["ci"] for p in analysis["curve"])
+
+
+def test_budget_stop_keeps_unrun_changes_in_instrument_denominator(pilot_case):
+    """A partial runner input index cannot make an unfinished pilot look complete."""
+    candidate, decision, kwargs = pilot_case
+    analysis = report.pilot_gain(candidate, decision, **{**kwargs, "budget": {"usd": 1, "calls": 1}})
+    assert not analysis["valid"]
+    assert analysis["instrument"]["changes"] == 4 and analysis["instrument"]["failures"] == 3
+    assert "after" not in analysis
+
+
+def test_report_restores_tool_program_and_reads_checkout(pilot_case, tmp_path):
+    """A saved ReAct candidate keeps its tools and reads source through the runner."""
+    from dspy.utils import DummyLM
+
+    _, decision, kwargs = pilot_case
+    _, _, candidate = candidate_at(tmp_path / "tool-proposal", tools=True)
+    decision = replace(decision, comparison={"candidate_id": candidate.program_id})
+    kwargs["lm"] = DummyLM(
+        [
+            answer
+            for _ in range(4)
+            for answer in (
+                {"next_thought": "Read", "next_tool_name": "read_file", "next_tool_args": {"path": "app.py"}},
+                {"next_thought": "Done", "next_tool_name": "finish", "next_tool_args": {}},
+                {"reasoning": "The interpreter receives input.", "findings": json.dumps([finding()])},
+            )
+        ]
+    )
+    analysis = report.pilot_gain(candidate, decision, **kwargs)
+    assert analysis["valid"] and analysis["after"]["pooled"]["search"]["recall"]["rate"] == 1
+    assert all("Failed to execute" not in str(entry.get("messages")) for entry in kwargs["lm"].history)
+
+
+def test_push_metrics_use_pilots_alternate_target_projection(pilot_case, population):
+    """A catch at an additional verified site counts once in the pilot's push metric."""
+    candidate, decision, kwargs = pilot_case
+    # This test isolates scoring projection; the remote primary site supplies no shown source lines.
+    kwargs["lm"] = model([[{**finding(), "cited_lines": []}], [], [{**finding(), "cited_lines": []}], []])
+    changes = []
+    for original in population[2]:
+        change = dict(original)
+        if change["kind"] == "vulnerability":
+            change.update(
+                path="other.py",
+                function="other",
+                function_lines=[40, 50],
+                files=[{"path": "app.py", "head_lines": [[40, 50]]}],
+                targets=[dict(path="app.py", function="run", function_lines=[1, 2], family="injection")],
+            )
+        changes.append(change)
+    analysis = report.pilot_gain(candidate, decision, **{**kwargs, "load_changes": lambda m, s: changes})
+    assert analysis["after"]["pooled"]["push"]["recall"]["count"] == 2
+    assert analysis["after"]["pooled"]["search"]["recall"]["count"] == 0

tests/test_hunt_rounds.py

@@ -0,0 +1,221 @@
+"""Score real fixture checkouts with scripted answers and compare separate folds."""
+
+import json
+import subprocess
+from copy import deepcopy
+from dataclasses import replace
+
+import pytest
+
+dspy = pytest.importorskip("dspy")
+pytest.importorskip("pandas")
+pytest.importorskip("statsmodels")
+from dspy.utils import DummyLM  # noqa: E402
+
+from openultrasast.hunt import accept, rounds, runner  # noqa: E402
+from openultrasast.hunt import program as hp  # noqa: E402
+from openultrasast.hunt.propose import Candidate  # noqa: E402
+from openultrasast.hunt.table import FeatureTable  # noqa: E402
+
+
+@pytest.fixture
+def population(tmp_path):
+    """Make two repository groups, each with a positive and an ordinary push."""
+    changes, rows, repos = [], [], {}
+    for fold in range(2):
+        root = tmp_path / f"repo-{fold}"
+        root.mkdir()
+
+        def git(*args, root=root):
+            return subprocess.run(
+                ["git", "-C", str(root), "-c", "core.hooksPath=/dev/null", *args], check=True, capture_output=True, text=True
+            ).stdout.strip()
+
+        git("init", "-q")
+        git("config", "user.name", "Fixture")
+        git("config", "user.email", "[email protected]")
+        (root / "app.py").write_text("def run():\n    return input()\n")
+        git("add", ".")
+        git("commit", "-qm", "base")
+        base = git("rev-parse", "HEAD")
+        (root / "app.py").write_text("def run():\n    return eval(input())\n")
+        git("add", ".")
+        git("commit", "-qm", "head")
+        head = git("rev-parse", "HEAD")
+        repo = f"fixture/repo-{fold}"
+        repos[repo] = str(root)
+        for kind in ("vulnerability", "ordinary"):
+            change = dict(
+                id=f"{fold}-{kind}",
+                repo=repo,
+                base=base,
+                head=head,
+                kind=kind,
+                family="injection",
+                path="app.py",
+                function="run",
+                function_lines=[1, 2],
+                case_id=f"case-{fold}-{kind}",
+                files=[{"path": "app.py", "head_lines": [[2, 2]]}],
+            )
+            changes.append(change)
+            rows.append(dict(change_id=change["id"], repo=repo, fold=fold))
+    pool = tmp_path / "pool-fixture.toml"
+    pool.write_text("status = 'fixture'\n")
+    return pool, repos, changes, FeatureTable(rows)
+
+
+def finding(confidence=0.9, family="injection"):
+    """A suspicion at the fixture's changed interpreter call."""
+    return dict(path="app.py", line=2, family=family, confidence=confidence, rationale="eval", cited_lines=[2])
+
+
+def model(answers):
+    """Return a local model whose answers go through ChatAdapter."""
+    return DummyLM([{"findings": json.dumps(answer)} for answer in answers])
+
+
+def candidate_at(out, *, tools=False):
+    """Save one instruction edit with the same structure as a real proposal."""
+    base = hp.HuntProgram(tools=hp.repository_tools(out) if tools else ())
+    program = base.deepcopy()
+    predictor = program.hunt.extract.predict if tools else program.hunt
+    predictor.signature = predictor.signature.with_instructions("Follow the changed interpreter call.")
+    candidate = Candidate(
+        hp.program_id(program),
+        "instruction",
+        {"instruction": "Follow the changed interpreter call."},
+        {},
+        deepcopy(program.dump_state()),
+        hp.program_id(base),
+        deepcopy(base.dump_state()),
+        (),
+        (),
+        str(out),
+    )
+    return base, program, candidate
+
+
+def score_args(population, out, lm):
+    """Supply the fixture manifest and its row folds to the runner wrapper."""
+    pool, repos, changes, table = population
+    return dict(
+        pool_manifest=pool,
+        repos=repos,
+        table=table,
+        folds=[0],
+        out_dir=out,
+        lm=lm,
+        budget={"usd": 1, "calls": 100},
+        load_changes=lambda m, s: changes,
+    )
+
+
+def test_fold_selection_repeat_resume_and_tools(population, tmp_path):
+    """Only selected folds run, and tools read each detached checkout on both repeats."""
+    base, _, _ = candidate_at(tmp_path, tools=True)
+    answers = []
+    for _ in range(4):
+        answers.extend(
+            [
+                {"next_thought": "Read the change", "next_tool_name": "read_file", "next_tool_args": {"path": "app.py"}},
+                {"next_thought": "Done", "next_tool_name": "finish", "next_tool_args": {}},
+                {"reasoning": "The changed call evaluates input.", "findings": json.dumps([finding()])},
+            ]
+        )
+    lm = DummyLM(answers)
+    kwargs = score_args(population, tmp_path / "folds", lm)
+    out = rounds.score_folds(base, program_id=hp.program_id(base), mask=(), repeats=2, **kwargs)
+    assert out == tmp_path / "folds"
+    for repeat in (0, 1):
+        entries = json.loads((out / str(repeat) / "inputs.json").read_text())
+        assert [e["item_id"] for e in entries] == [c["id"] for c in population[2][:2]]
+        record = json.loads(runner.record_path(out / str(repeat), entries[0]["item_id"]).read_text())
+        assert record["instrument"]["changed_bytes"]["app.py"] > 0
+        assert record["instrument"]["base_verified"] and record["instrument"]["head_verified"]
+        assert record["push"]["admission"]["defects"]
+    assert any("eval(input())" in str(entry.get("messages")) for entry in lm.history)
+    assert 2 in base.seen["app.py"]
+    calls = len(lm.history)
+    rounds.score_folds(base, program_id=hp.program_id(base), mask=(), repeats=2, **kwargs)
+    assert len(lm.history) == calls
+
+
+def test_fold_refuses_pilot_and_conflicting_rows(population, tmp_path):
+    """Fold selection cannot bypass the pilot guard or split repository groups."""
+    pool, _, _, table = population
+    kwargs = score_args(population, tmp_path / "folds", model([]))
+    (pool.with_name("freeze-fixture.json")).write_text(json.dumps({"freeze_digest": runner.PILOT_FREEZE_DIGEST}))
+    with pytest.raises(ValueError, match="held out"):
+        rounds.score_folds(hp.HuntProgram(), program_id="p", mask=(), **kwargs)
+    pool.with_name("freeze-fixture.json").unlink()
+    table.rows[1]["fold"] = 1
+    with pytest.raises(ValueError, match="repository"):
+        rounds.score_folds(hp.HuntProgram(), program_id="p", mask=(), **kwargs)
+
+
+def test_fold_results_feed_decide_and_outcomes_flip_it(population, tmp_path):
+    """Paired repository estimates accept an improvement and reject the reversed outcomes."""
+    base, program, candidate = candidate_at(tmp_path / "candidate")
+    for name, prog, answers in (("current", base, [[]] * 8), ("candidate", program, [[finding()], []] * 4)):
+        kwargs = score_args(population, tmp_path / name, model(answers))
+        rounds.score_folds(prog, program_id=hp.program_id(prog), mask=(), repeats=2, **{**kwargs, "folds": [0, 1]})
+    current, proposed = tmp_path / "current", tmp_path / "candidate"
+
+    def results(a, b, scope):
+        return rounds.fold_results(a, b, population[2], scope=scope, tolerance=10, seed=13, resamples=50)
+
+    compiled = results(current, proposed, "compile")
+    held = results(current, proposed, "holdout")
+    assert compiled["pooled"].recall.diff == 1
+    assert compiled["pooled"].recall.units == 4 and compiled["pooled"].recall.groups == 2
+    assert compiled["injection"].suspicions_per_push.diff == 0
+    rule = accept.Rule(0, {}, 10000)
+    assert accept.decide(candidate.base_program_id, candidate, compiled, held, rule).accepted
+    assert not accept.decide(
+        candidate.base_program_id, candidate, results(proposed, current, "compile"), results(proposed, current, "holdout"), rule
+    ).accepted
+    # Each failure mode retains a positive unit but removes stale findings.
+    for repeat, change, field in ((0, population[2][0], "parse_failed"), (1, population[2][0], "status")):
+        path = runner.record_path(proposed / str(repeat), change["id"])
+        record = json.loads(path.read_text())
+        record[field] = True if field == "parse_failed" else "instrument_failure"
+        path.write_text(json.dumps(record))
+    runner.record_path(proposed / "0", population[2][2]["id"]).unlink()
+    remaining = results(current, proposed, "compile")["pooled"].recall
+    assert remaining.units == 4 and remaining.diff == 0.25
+    changed_folds = [{**c, "fold": 99} for c in population[2]]
+    with pytest.raises(ValueError, match="saved study"):
+        rounds.fold_results(current, proposed, changed_folds, scope="compile", tolerance=10, seed=13, resamples=50)
+
+
+def test_round_trip_scores_four_passes_and_records_rejection(population, tmp_path):
+    """Compile and holdout run separately; a rejected tie reaches the proposal history."""
+    base, _, candidate = candidate_at(tmp_path / "proposal")
+    kwargs = score_args(population, tmp_path / "round", model([[]] * 8))
+    kwargs.pop("folds")
+    decision = rounds.round_trip(
+        base,
+        candidate,
+        current_id=candidate.base_program_id,
+        compile_folds=[0],
+        holdout_folds=[1],
+        rule=accept.Rule(0, {}, 10000),
+        seed=13,
+        resamples=50,
+        **kwargs,
+    )
+    assert not decision.accepted and "holdout_tie" in decision.reasons
+    assert len(list((tmp_path / "round").glob("*/*/0/inputs.json"))) == 4
+    rejected = json.loads((tmp_path / "proposal" / "rejected.jsonl").read_text())
+    assert rejected["program_id"] == candidate.program_id and "holdout_results" not in rejected
+    with pytest.raises(ValueError, match="disjoint"):
+        rounds.round_trip(
+            base,
+            replace(candidate),
+            current_id=candidate.base_program_id,
+            compile_folds=[0],
+            holdout_folds=[0],
+            rule=accept.Rule(0, {}, 10000),
+            **kwargs,
+        )