feat(hunt): proposals and the acceptance rule (5.1, 5.2)

propose.py: one change per round (instruction, demonstrations or one mask), examples built from
compile folds only with a provenance record of the change ids used, a registry for GEPA, MIPROv2
and BootstrapFewShot (MIPROv2 without the data-aware proposer so labels are never serialised),
compile-fold scoring with Wilson and the repository-paired bootstrap, and a compile-only history
the next round may read. accept.py: the rule as independent clauses (both folds non-negative with
one strictly positive, per-family regression budget on held-out recall, tie tolerance, no
acceptance without an interval, instruction cap, one change only), the comparison recorded in the
candidate artifact's configurations, rejections logged without held-out numbers. 48 fixture tests.
Known gap for live rounds: the optimisers and propose._score call the program directly, so a
tool-using program must be scored through the runner with its tools bound per checkout (6.1).
Implemented by Codex (gpt-6-astra), gated here.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/test_hunt_accept.py

@@ -0,0 +1,164 @@
+"""Each acceptance clause has an independent numerical counterexample."""
+
+import json
+from copy import deepcopy
+from dataclasses import replace
+
+import pytest
+
+from openultrasast.hunt.accept import FoldResult, Rule, decide, record_rejection
+from openultrasast.hunt.artifact import RunArtifact
+from openultrasast.hunt.propose import Candidate
+
+
+def estimate(diff=0.2):
+    """A synthetic paired estimate with counts and a finite interval."""
+    return {"diff": diff, "ci": [diff - 0.1, diff + 0.1], "units": 20, "groups": 4}
+
+
+@pytest.fixture
+def candidate(tmp_path):
+    """A single instruction edit, with its baseline available for comparison."""
+    base = {"hunt": {"signature": {"instructions": "old", "fields": []}, "demos": []}}
+    state = deepcopy(base)
+    state["hunt"]["signature"]["instructions"] = "new"
+    return Candidate(
+        "candidate",
+        "instruction",
+        {"instruction": {"hunt": "new"}},
+        {"recall": 0.7, "ci": [0.5, 0.9]},
+        state,
+        "current",
+        base,
+        (),
+        (),
+        str(tmp_path),
+    )
+
+
+@pytest.fixture
+def results():
+    """Two scopes containing pooled and family differences."""
+    return (
+        {family: FoldResult("compile", family, estimate(), estimate(0)) for family in ("pooled", "injection")},
+        {family: FoldResult("holdout", family, estimate(), estimate(0)) for family in ("pooled", "injection")},
+    )
+
+
+def rule(**kwargs):
+    """A fixed rule with no permitted family regression by default."""
+    return Rule(**({"tolerance": 0.01, "regression_budget": {"default": 0, "injection": -0.1}, "instruction_cap": 100} | kwargs))
+
+
+def difference(results, scope, family, value):
+    """Change one measured recall difference."""
+    results[scope][family] = replace(results[scope][family], recall=estimate(value))
+
+
+@pytest.mark.parametrize("scope,reason", [(0, "compile_recall_negative"), (1, "holdout_recall_negative")])
+def test_negative_pooled_difference_rejects(candidate, results, scope, reason):
+    """Either negative pooled result defeats an otherwise acceptable proposal."""
+    assert decide("current", candidate, *results, rule()).accepted
+    difference(results, scope, "pooled", -0.01)
+    decision = decide("current", candidate, *results, rule())
+    assert not decision.accepted and reason in decision.reasons
+
+
+def test_both_zero_rejects(candidate, results):
+    """Zero gain in both scopes is insufficient and is a held-out tie."""
+    difference(results, 0, "pooled", 0)
+    difference(results, 1, "pooled", 0)
+    decision = decide("current", candidate, *results, rule())
+    assert not decision.accepted
+    assert {"no_positive_recall_difference", "holdout_tie"} <= set(decision.reasons)
+
+
+def test_compile_zero_holdout_positive_accepts(candidate, results):
+    """A positive held-out gain can accept when compile recall is unchanged."""
+    difference(results, 0, "pooled", 0)
+    assert decide("current", candidate, *results, rule()).accepted
+
+
+def test_heldout_tolerance_rejects_even_with_compile_gain(candidate, results):
+    """The tolerance boundary is inclusive and applies to held-out recall."""
+    difference(results, 1, "pooled", 0.01)
+    assert "holdout_tie" in decide("current", candidate, *results, rule()).reasons
+    difference(results, 1, "pooled", 0.0101)
+    assert decide("current", candidate, *results, rule()).accepted
+
+
+def test_family_budget_overrides_pooled_gain(candidate, results):
+    """A family at its allowed floor passes; below it rejects."""
+    difference(results, 1, "injection", -0.1)
+    assert decide("current", candidate, *results, rule()).accepted
+    difference(results, 1, "injection", -0.101)
+    assert "family_regression:injection" in decide("current", candidate, *results, rule()).reasons
+
+
+def test_unlisted_family_uses_default_budget(candidate, results):
+    """Families without an override still have a regression floor."""
+    for scope, name in enumerate(("compile", "holdout")):
+        results[scope]["path"] = FoldResult(name, "path", estimate(-0.01), estimate(0))
+    assert "family_regression:path" in decide("current", candidate, *results, rule()).reasons
+    assert decide("current", candidate, *results, rule(regression_budget={"default": -0.02})).accepted
+
+
+def test_instruction_cap_uses_saved_instruction(candidate, results):
+    """The state is authoritative even when the reported change is short."""
+    assert decide("current", candidate, *results, rule(instruction_cap=3)).accepted
+    decision = decide("current", candidate, *results, rule(instruction_cap=2))
+    assert not decision.accepted and "instruction_cap" in decision.reasons
+
+
+@pytest.mark.parametrize("scope,family,metric", [(0, "pooled", "recall"), (1, "injection", "recall"), (1, "pooled", "suspicions_per_push")])
+def test_missing_interval_rejects(candidate, results, scope, family, metric):
+    """Neither a family nor a suspicion point estimate can stand in for an interval."""
+    result = results[scope][family]
+    results[scope][family] = replace(result, **{metric: {**getattr(result, metric), "ci": None}})
+    assert not decide("current", candidate, *results, rule()).accepted
+    assert any(r.startswith("missing_interval:") for r in decide("current", candidate, *results, rule()).reasons)
+
+
+def test_multiple_changes_reject_even_when_kind_claims_instruction(candidate, results):
+    """Saved demos reveal an undeclared second edit."""
+    state = deepcopy(candidate.state)
+    state["hunt"]["demos"] = [{"findings": []}]
+    decision = decide("current", replace(candidate, state=state), *results, rule())
+    assert not decision.accepted and "multiple_or_invalid_changes" in decision.reasons
+
+
+def test_comparison_is_saved_and_fits_existing_artifact(candidate, results, tmp_path):
+    """Both IDs, scopes and the rule survive an artifact round trip."""
+    decision = decide("current", candidate, *results, rule())
+    assert decision.comparison["current_id"] == "current"
+    assert decision.comparison["candidate_id"] == "candidate"
+    assert decision.comparison["compile_results"]["pooled"]["recall"]["diff"] == 0.2
+    assert decision.comparison["holdout_results"]["injection"]["recall"]["ci"]
+    assert decision.comparison["rule"]["version"] == "accept-v1"
+    assert json.loads((tmp_path / "comparison.json").read_text()) == decision.comparison
+    artifact = RunArtifact("candidate", 1, (), {}, [], {}, {}, "fixture")
+    decision.record(tmp_path, artifact=artifact)
+    assert artifact.configurations == [{"comparison": decision.comparison}]
+    path = artifact.write(tmp_path)
+    assert RunArtifact.read(path).configurations == artifact.configurations
+
+
+def test_rejection_omits_heldout_numbers(candidate, results, tmp_path):
+    """The rejection journal discloses reasons and compile measurements only."""
+    difference(results, 1, "pooled", -0.987654)
+    decision = decide("current", candidate, *results, rule())
+    record_rejection(tmp_path, candidate, decision)
+    record_rejection(tmp_path, candidate, decision)
+    rows = [json.loads(line) for line in (tmp_path / "rejected.jsonl").read_text().splitlines()]
+    assert len(rows) == 2 and rows[0]["reasons"] == decision.reasons
+    assert rows[0]["compile_score"] == candidate.compile_score
+    assert "-0.987654" not in json.dumps(rows)
+    assert set(rows[0]) == {"program_id", "kind", "change", "compile_score", "reasons"}
+
+
+def test_missing_family_and_wrong_scope_cannot_accept(candidate, results):
+    """Incomplete or mislabelled comparisons are not usable evidence."""
+    del results[1]["injection"]
+    assert not decide("current", candidate, *results, rule()).accepted
+    results[1]["pooled"] = replace(results[1]["pooled"], scope="compile")
+    assert not decide("current", candidate, *results, rule()).accepted