feat(hunt): proposals and the acceptance rule (5.1, 5.2)

propose.py: one change per round (instruction, demonstrations or one mask), examples built from
compile folds only with a provenance record of the change ids used, a registry for GEPA, MIPROv2
and BootstrapFewShot (MIPROv2 without the data-aware proposer so labels are never serialised),
compile-fold scoring with Wilson and the repository-paired bootstrap, and a compile-only history
the next round may read. accept.py: the rule as independent clauses (both folds non-negative with
one strictly positive, per-family regression budget on held-out recall, tie tolerance, no
acceptance without an interval, instruction cap, one change only), the comparison recorded in the
candidate artifact's configurations, rejections logged without held-out numbers. 48 fixture tests.
Known gap for live rounds: the optimisers and propose._score call the program directly, so a
tool-using program must be scored through the runner with its tools bound per checkout (6.1).
Implemented by Codex (gpt-6-astra), gated here.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

.kiro/specs/hunter-learning-loop/tasks.md

@@ -77,12 +77,12 @@
   - Done when a fixture run is marked contaminated after its repository is added to a report list
   - _Requirements: 2.7_
 
-- [ ] 5. Core: proposals and acceptance
-- [ ] 5.1 Propose candidates with the program runtime's optimisers
+- [x] 5. Core: proposals and acceptance
+- [x] 5.1 Propose candidates with the program runtime's optimisers
   - One change per round: an instruction proposal, a demonstration set, or a mask; proposals score on compile folds with the search metric; the proposer never sees held-out results; rejected candidates stay visible to the next round
   - Done when a round on a fixture produces a candidate with its compile-fold score and the proposer's inputs are shown to exclude held-out data
   - _Requirements: 5.4, 6.6, 6.8_
-- [ ] 5.2 Accept or reject by the rule
+- [x] 5.2 Accept or reject by the rule
   - Compile-fold and held-out results per family; accepted only when both are non-negative, one strictly positive, no family below its regression budget, instruction under the length cap; ties rejected; the comparison is recorded in the new artifact
   - Done when every clause of the rule has a synthetic fold-result test that flips the decision
   - _Requirements: 5.6, 6.1, 6.2, 6.3, 6.4, 6.5, 6.7_

src/openultrasast/hunt/accept.py

@@ -0,0 +1,180 @@
+"""Accept a proposal only when separate compile and held-out comparisons support it."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from dataclasses import asdict, dataclass, is_dataclass
+from pathlib import Path
+from typing import Any
+
+from ..stats import Estimate
+from .artifact import RunArtifact
+from .propose import Candidate, _append, _write, changed_surfaces, proposer_record
+
+
+@dataclass(frozen=True)
+class Rule:
+    """The acceptance rule fixed before measurement.
+
+    Regression budgets are signed recall differences: -0.02 permits a loss of
+    two percentage points. The 'default' entry applies to unlisted families;
+    when omitted it is zero. The held-out tie band includes both endpoints.
+    """
+
+    tolerance: float
+    regression_budget: Mapping[str, float]
+    instruction_cap: int
+    version: str = "accept-v1"
+
+    def __post_init__(self) -> None:
+        """Reject malformed or unsupported rules before using any measurements."""
+        if self.version != "accept-v1":
+            raise ValueError("unsupported acceptance rule version")
+        if not math.isfinite(self.tolerance) or self.tolerance < 0:
+            raise ValueError("tolerance must be finite and non-negative")
+        if type(self.instruction_cap) is not int or self.instruction_cap < 0:
+            raise ValueError("instruction cap must be a non-negative integer")
+        if any(not math.isfinite(v) or not -1 <= v <= 0 for v in self.regression_budget.values()):
+            raise ValueError("regression budgets must be signed differences between -1 and zero")
+
+
+@dataclass(frozen=True)
+class FoldResult:
+    """Paired candidate-minus-current differences for one scope and one family.
+
+    Use family='pooled' for the aggregate and scope='compile' or 'holdout'. Each
+    metric accepts stats.Estimate or a record containing diff, ci, units and
+    groups. Missing intervals remain explicit so decide can reject them.
+    """
+
+    scope: str
+    family: str
+    recall: Estimate | Mapping[str, Any]
+    suspicions_per_push: Estimate | Mapping[str, Any]
+
+
+@dataclass(frozen=True)
+class Decision:
+    """The verdict, each failed clause, and the full private comparison."""
+
+    accepted: bool
+    reasons: list[str]
+    comparison: dict[str, Any]
+
+    def record(self, out_dir: Path | str, *, artifact: RunArtifact | None = None) -> None:
+        """Save comparison.json and optionally attach it to the candidate's artifact.
+
+        The existing configurations field carries the full comparison. Call
+        artifact.write afterwards to persist its new content address.
+        """
+        if artifact is not None and artifact.program_id != self.comparison["candidate_id"]:
+            raise ValueError("comparison belongs to a different program artifact")
+        _write(Path(out_dir) / "comparison.json", self.comparison)
+        if artifact is not None:
+            record = {"comparison": self.comparison}
+            if record not in artifact.configurations:
+                artifact.configurations.append(record)
+
+
+def _record(value: Any) -> dict[str, Any]:
+    """Convert dataclass estimates and ordinary mappings to JSON records."""
+    return asdict(value) if is_dataclass(value) else dict(value)
+
+
+def _valid_number(value: Any) -> bool:
+    """A missing or non-finite point estimate cannot support acceptance."""
+    return isinstance(value, int | float) and not isinstance(value, bool) and math.isfinite(value)
+
+
+def _interval_present(estimate: Mapping[str, Any]) -> bool:
+    """Require an ordered finite interval and real unit and group counts."""
+    ci = estimate.get("ci")
+    return (
+        isinstance(ci, tuple | list)
+        and len(ci) == 2
+        and all(_valid_number(v) for v in ci)
+        and ci[0] <= ci[1]
+        and _valid_number(estimate.get("diff"))
+        and type(estimate.get("units")) is int
+        and type(estimate.get("groups")) is int
+        and 0 < estimate["groups"] <= estimate["units"]
+    )
+
+
+def decide(
+    current_id: str, candidate: Candidate, compile_results: Mapping[str, FoldResult], holdout_results: Mapping[str, FoldResult], rule: Rule
+) -> Decision:
+    """Apply every clause independently and save the comparison beside the proposal.
+
+    A zero compile difference and positive held-out difference may accept. A
+    positive compile difference with a held-out tie rejects. Confidence
+    intervals must exist; they need not exclude zero under this rule.
+    """
+    rule.__post_init__()
+    reasons = []
+    if candidate.base_program_id != current_id:
+        reasons.append("current_program_mismatch")
+    try:
+        surfaces = changed_surfaces(candidate.base_state, candidate.state, candidate.base_mask, candidate.mask)
+        if surfaces != {candidate.kind} or set(candidate.change) != {candidate.kind}:
+            reasons.append("multiple_or_invalid_changes")
+    except ValueError:
+        reasons.append("multiple_or_invalid_changes")
+    instructions = [state.get("signature", {}).get("instructions", "") for state in candidate.state.values()]
+    if any(not isinstance(text, str) or len(text) > rule.instruction_cap for text in instructions):
+        reasons.append("instruction_cap")
+    compiled, heldout = {}, {}
+    for scope, source, target in (("compile", compile_results, compiled), ("holdout", holdout_results, heldout)):
+        for family, result in source.items():
+            record = _record(result)
+            if record.get("scope") != scope or record.get("family") != family:
+                reasons.append(f"invalid_scope:{scope}:{family}")
+            target[family] = record
+            for metric in ("recall", "suspicions_per_push"):
+                estimate = _record(record.get(metric, {}))
+                record[metric] = estimate
+                if not _interval_present(estimate):
+                    reasons.append(f"missing_interval:{scope}:{family}:{metric}")
+        if "pooled" not in source:
+            reasons.append(f"missing_pooled:{scope}")
+    if set(compiled) != set(heldout):
+        reasons.append("missing_family_comparison")
+    compile_diff = compiled.get("pooled", {}).get("recall", {}).get("diff")
+    holdout_diff = heldout.get("pooled", {}).get("recall", {}).get("diff")
+    if _valid_number(compile_diff) and compile_diff < 0:
+        reasons.append("compile_recall_negative")
+    if _valid_number(holdout_diff) and holdout_diff < 0:
+        reasons.append("holdout_recall_negative")
+    if not any(_valid_number(value) and value > 0 for value in (compile_diff, holdout_diff)):
+        reasons.append("no_positive_recall_difference")
+    if _valid_number(holdout_diff) and abs(holdout_diff) <= rule.tolerance:
+        reasons.append("holdout_tie")
+    for family, result in heldout.items():
+        if family == "pooled":
+            continue
+        diff = result["recall"].get("diff")
+        floor = rule.regression_budget.get(family, rule.regression_budget.get("default", 0))
+        if _valid_number(diff) and diff < floor:
+            reasons.append(f"family_regression:{family}")
+    comparison = {
+        "current_id": current_id,
+        "candidate_id": candidate.program_id,
+        "compile_results": compiled,
+        "holdout_results": heldout,
+        "rule": asdict(rule),
+        "accepted": not reasons,
+        "reasons": reasons,
+    }
+    decision = Decision(not reasons, reasons, comparison)
+    decision.record(candidate.out_dir)
+    return decision
+
+
+def record_rejection(out_dir: Path | str, candidate: Candidate, decision: Decision) -> None:
+    """Append the compile score and reason codes, never the private comparison."""
+    if decision.accepted:
+        raise ValueError("cannot record an accepted candidate as rejected")
+    if decision.comparison["candidate_id"] != candidate.program_id:
+        raise ValueError("decision belongs to a different candidate")
+    _append(Path(out_dir) / "rejected.jsonl", {**proposer_record(candidate), "reasons": decision.reasons})

src/openultrasast/hunt/propose.py

@@ -0,0 +1,349 @@
+"""Propose one program change using compile folds and keep a compile-only history.
+
+Pass an optimiser name, a configured DSPy optimiser, or a callable taking
+(program, examples, metric) and returning a compiled program. Callables receive
+copies: neither the baseline nor the labels used for scoring can be edited.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from copy import deepcopy
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+from .. import stats
+from .metric import search_metric, search_outcome
+from .table import FEATURE_COLUMNS, canonical_json
+
+
+class _Optimisers(Mapping):
+    """Load a DSPy optimiser class only when its registry entry is requested."""
+
+    names = {"gepa": "GEPA", "miprov2": "MIPROv2", "bootstrap": "BootstrapFewShot"}
+
+    def __getitem__(self, name: str) -> type:
+        import dspy
+
+        return getattr(dspy, self.names[name])
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self.names)
+
+    def __len__(self) -> int:
+        return len(self.names)
+
+
+OPTIMISERS = _Optimisers()
+
+
+@dataclass(frozen=True)
+class Round:
+    """Register one editable surface, the compile folds, and the run's bounds."""
+
+    kind: str
+    base_program_id: str
+    compile_folds: list[int]
+    seed: int
+    optimiser: str
+    budget: Mapping[str, float]
+
+    def __post_init__(self) -> None:
+        """Refuse ambiguous kinds, empty folds, and invalid budget bounds."""
+        if self.kind not in ("instruction", "demos", "mask"):
+            raise ValueError("round kind must name exactly one of instruction, demos or mask")
+        if not self.compile_folds or any(type(f) is not int or f < 0 for f in self.compile_folds):
+            raise ValueError("compile folds must be nonempty non-negative integers")
+        if len(set(self.compile_folds)) != len(self.compile_folds):
+            raise ValueError("compile folds cannot repeat")
+        if set(self.budget) != {"usd", "calls"} or any(not math.isfinite(v) or v < 0 for v in self.budget.values()):
+            raise ValueError("budget requires finite non-negative usd and calls")
+        if int(self.budget["calls"]) != self.budget["calls"]:
+            raise ValueError("call budget must be an integer")
+
+
+class CompileExamples(list):
+    """A list of DSPy examples with the change and fold provenance alongside it."""
+
+    @property
+    def provenance(self) -> list[dict[str, Any]]:
+        """Return only the identifiers needed to audit the proposer's population."""
+        return [{"change_id": e.change_id, "fold": e.fold} for e in self]
+
+
+def compile_examples(
+    changes: Sequence[Mapping[str, Any]],
+    rows_by_change: Mapping[str, Sequence[Mapping[str, Any]]],
+    *,
+    compile_folds: Sequence[int],
+    render: Callable,
+) -> list:
+    """Build examples only for compile folds, with scoring labels outside inputs.
+
+    Render takes (change, rows) and calls program.render_rows with source texts
+    bound by the caller, just as runner.hunt_change supplies its checkout texts.
+    It receives only feature rows and no advisory or verification metadata.
+    The returned list's provenance property is written by propose to
+    proposer_inputs.json. No held-out rows are accessed or rendered.
+    """
+    import dspy
+
+    from .program import INPUTS
+
+    repo_folds, ids = {}, set()
+    for change in changes:
+        if change["id"] in ids:
+            raise ValueError("change IDs must be unique")
+        ids.add(change["id"])
+        if change["repo"] in repo_folds and repo_folds[change["repo"]] != change["fold"]:
+            raise ValueError("a repository cannot cross folds")
+        repo_folds[change["repo"]] = change["fold"]
+    examples = CompileExamples()
+    for change in changes:
+        if change["fold"] not in compile_folds:
+            continue
+        rows = rows_by_change[change["id"]]
+        if any(row.get("fold", change["fold"]) != change["fold"] for row in rows):
+            raise ValueError("row folds disagree with the change")
+        safe_rows = [{k: deepcopy(v) for k, v in row.items() if k in (*FEATURE_COLUMNS, "path", "line")} for row in rows]
+        inputs = render({"id": change["id"]}, safe_rows)
+        if set(inputs) - {name for name, _, _ in INPUTS} or any(not isinstance(v, str) for v in inputs.values()):
+            raise ValueError("renderer must return only the hunt signature's text inputs")
+        metric_change = {k: deepcopy(change[k]) for k in ("kind", "family", "path", "function", "function_lines") if k in change}
+        metric_change["files"] = [
+            {k: deepcopy(file[k]) for k in ("path", "head_lines", "function_lines") if k in file} for file in change.get("files", [])
+        ]
+        examples.append(
+            dspy.Example(**inputs, change=metric_change, change_id=change["id"], repo=change["repo"], fold=change["fold"]).with_inputs(
+                *inputs
+            )
+        )
+    return examples
+
+
+@dataclass(frozen=True)
+class Candidate:
+    """A saved proposal and compile evidence, with enough baseline state to audit edits."""
+
+    program_id: str
+    kind: str
+    change: dict[str, Any]
+    compile_score: dict[str, Any]
+    state: dict[str, Any]
+    base_program_id: str
+    base_state: dict[str, Any]
+    mask: tuple[str, ...]
+    base_mask: tuple[str, ...]
+    out_dir: str
+
+
+def changed_surfaces(base: Mapping, candidate: Mapping, base_mask: Sequence[str], mask: Sequence[str]) -> set[str]:
+    """Compare saved predictor states, permitting only instructions, demos or one mask.
+
+    Signature field removal or addition is part of masking. Every surviving
+    field must retain its description. Other saved settings cannot be edited.
+    """
+    from .program import INPUTS, signature_for
+
+    before, after = deepcopy(dict(base)), deepcopy(dict(candidate))
+    changed = set()
+    mask_changed = set(base_mask) != set(mask)
+    if mask_changed:
+        allowed = {name for name, _, _ in INPUTS} | set(FEATURE_COLUMNS) | {"diff", "source", "semantic", "taxonomy"}
+        if (set(mask) | set(base_mask)) - allowed or len(set(base_mask) ^ set(mask)) != 1:
+            raise ValueError("one mask annotation may change per round")
+        changed.add("mask")
+    if set(before) != set(after):
+        raise ValueError("predictor structure is not an editable surface")
+    for name in before:
+        left, right = before[name], after[name]
+        for key, kind in (("demos", "demos"),):
+            if left.pop(key, []) != right.pop(key, []):
+                changed.add(kind)
+        ls, rs = left.get("signature", {}), right.get("signature", {})
+        if ls.pop("instructions", None) != rs.pop("instructions", None):
+            changed.add("instruction")
+        if mask_changed:
+            # Check the exact expected field transition, including ReAct's extra fields.
+            old = signature_for(base_mask).dump_state()["fields"]
+            new = signature_for(mask).dump_state()["fields"]
+            removed = [field for field in old if field not in new]
+            added = [field for field in new if field not in old]
+            lf, rf = ls.get("fields", []), rs.get("fields", [])
+            if [f for f in lf if f not in removed] != [f for f in rf if f not in added]:
+                raise ValueError("mask changed unrelated signature fields")
+            if any(f not in lf for f in removed) or any(f not in rf for f in added):
+                raise ValueError("mask does not match signature fields")
+            ls.pop("fields", None)
+            rs.pop("fields", None)
+    if before != after:
+        raise ValueError("only instructions, demos or mask may change")
+    return changed
+
+
+def _change(kind: str, state: Mapping, examples: Sequence, mask: Sequence[str]) -> dict[str, Any]:
+    """Describe the actual edit, tracing every demonstration to compile inputs."""
+    if kind == "instruction":
+        return {"instruction": {name: value["signature"]["instructions"] for name, value in state.items()}}
+    if kind == "mask":
+        return {"mask": list(mask)}
+    demos = [demo for value in state.values() for demo in value.get("demos", [])]
+    ids = set()
+    for demo in demos:
+        matches = [e.change_id for e in examples if all(demo.get(k) == v for k, v in e.inputs().items())]
+        if not matches:
+            raise ValueError("demonstrations must come from compile examples")
+        ids.update(matches)
+    return {"demos": {"count": len(demos), "change_ids": sorted(ids)}}
+
+
+def _score(program: Any, examples: Sequence) -> list[dict[str, Any]]:
+    """Run each example once with inputs limited to this program's mask."""
+    from .program import signature_for
+
+    wanted = signature_for(program.mask).input_fields
+    outcomes = []
+    for example in examples:
+        inputs = {k: v for k, v in example.inputs().items() if k in wanted}
+        if set(inputs) != set(wanted):
+            raise ValueError("examples lack an unmasked input; render full inputs before proposing")
+        prediction = program(**inputs)
+        outcome = search_outcome(example.change, prediction.findings)
+        outcomes.append(
+            {"change_id": example.change_id, "repo": example.repo, "fold": example.fold, "kind": example.change["kind"], **asdict(outcome)}
+        )
+    return outcomes
+
+
+def _compile(round: Round, program: Any, examples: list, optimiser: Any, metric: Callable) -> Any:
+    """Adapt the named optimisers without ever supplying a held-out validation set."""
+    import dspy
+
+    if isinstance(optimiser, str):
+        if optimiser != round.optimiser:
+            raise ValueError("optimiser differs from the registered round")
+        cls = OPTIMISERS[optimiser]
+        if optimiser == "gepa":
+            if dspy.settings.lm is None:
+                raise ValueError("GEPA needs a configured reflection model in dspy.context")
+
+            def feedback(example, prediction, trace=None, pred_name=None, pred_trace=None):
+                """Give GEPA the same score and the metric's compile-only feedback."""
+                import dspy
+
+                outcome = search_outcome(example.change, prediction.findings)
+                return dspy.Prediction(score=metric(example, prediction, trace), feedback=outcome.feedback)
+
+            optimiser = cls(metric=feedback, max_metric_calls=int(round.budget["calls"]), seed=round.seed, reflection_lm=dspy.settings.lm)
+        elif optimiser == "miprov2":
+            optimiser = cls(metric=metric, seed=round.seed, max_bootstrapped_demos=0, max_labeled_demos=0)
+        else:
+            optimiser = cls(metric=metric, max_labeled_demos=0)
+    if isinstance(optimiser, dspy.MIPROv2):
+        # DSPy's dataset summary serializes whole examples, including metric labels.
+        return optimiser.compile(program, trainset=examples, data_aware_proposer=False)
+    if hasattr(optimiser, "compile"):
+        return optimiser.compile(program, trainset=examples)
+    return optimiser(program, examples, metric)
+
+
+def propose(
+    round: Round, program: Any, examples: list, *, optimiser: Any, metric: Callable = search_metric, out_dir: Path | str
+) -> Candidate:
+    """Compile a copy, validate its one edit, score against the base, and save it.
+
+    The saved state is candidates/<program_id>.json. The history contains only
+    the edit and compile score; comparisons and held-out measurements are kept
+    separately by accept. A callable optimiser must respect the registered
+    money and call bounds; these are recorded in round.json for audit.
+    """
+    from .program import program_id
+
+    round.__post_init__()
+    base_mask = tuple(program.mask)
+    if round.base_program_id != program_id(program, mask=base_mask):
+        raise ValueError("round base program ID does not match")
+    if not examples or any(e.fold not in round.compile_folds for e in examples):
+        raise ValueError("proposer examples must come only from compile folds")
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    _write(out / "proposer_inputs.json", [{"change_id": e.change_id, "fold": e.fold} for e in examples])
+    _write(out / "round.json", asdict(round))
+    base = program.deepcopy()
+    scoring_examples = deepcopy(examples)
+    state = deepcopy(base.dump_state())
+    compiled = _compile(round, program.deepcopy(), deepcopy(scoring_examples), optimiser, metric)
+    mask = tuple(compiled.mask)
+    candidate_state = deepcopy(compiled.dump_state())
+    if changed_surfaces(state, candidate_state, base_mask, mask) != {round.kind}:
+        raise ValueError("a round must change exactly one declared surface")
+    change = _change(round.kind, candidate_state, scoring_examples, mask)
+    base_outcomes = _score(base, scoring_examples)
+    outcomes = _score(compiled, scoring_examples)
+    import pandas as pd
+
+    paired = {}
+    for kind, field, name in (("vulnerability", "hit", "recall"), ("ordinary", "suspicions", "suspicions_per_push")):
+        frame = pd.DataFrame(
+            [
+                {"repo": b["repo"], "base": a[field], "candidate": b[field]}
+                for a, b in zip(base_outcomes, outcomes, strict=True)
+                if b["kind"] == kind
+            ],
+            columns=["repo", "base", "candidate"],
+        )
+        paired[name] = asdict(stats.paired_bootstrap(frame, "repo", "base", "candidate", seed=round.seed))
+    positives = [o for o in outcomes if o["kind"] == "vulnerability"]
+    hits = sum(o["hit"] for o in positives)
+    score = {
+        "recall": hits / len(positives) if positives else None,
+        "ci": stats.wilson(hits, len(positives)),
+        "hits": hits,
+        "positive_units": len(positives),
+        "ordinary_units": len(outcomes) - len(positives),
+        "paired": paired,
+        "outcomes": outcomes,
+        "base_outcomes": base_outcomes,
+    }
+    candidate = Candidate(
+        program_id(compiled, mask=mask), round.kind, change, score, candidate_state, round.base_program_id, state, mask, base_mask, str(out)
+    )
+    _write(out / "candidates" / f"{candidate.program_id}.json", asdict(candidate))
+    _append(out / "candidates.jsonl", proposer_record(candidate))
+    return candidate
+
+
+def proposer_record(candidate: Candidate) -> dict[str, Any]:
+    """Return only the fields that belong in the compile-only history."""
+    return {name: deepcopy(getattr(candidate, name)) for name in ("program_id", "kind", "change", "compile_score")}
+
+
+def visible_to_proposer(out_dir: Path | str) -> list[dict[str, Any]]:
+    """Read every past proposal, irrespective of acceptance, without reading comparisons."""
+    path = Path(out_dir) / "candidates.jsonl"
+    if not path.exists():
+        return []
+    return [
+        {key: record[key] for key in ("program_id", "kind", "change", "compile_score")}
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+        for record in [json.loads(line)]
+    ]
+
+
+def _write(path: Path, value: Any) -> None:
+    """Replace one JSON record atomically."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".partial")
+    temporary.write_text(canonical_json(value) + "\n", encoding="utf-8")
+    temporary.replace(path)
+
+
+def _append(path: Path, value: Any) -> None:
+    """Append a complete JSON line to the selected journal."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as stream:
+        stream.write(canonical_json(value) + "\n")

tests/test_hunt_accept.py

@@ -0,0 +1,164 @@
+"""Each acceptance clause has an independent numerical counterexample."""
+
+import json
+from copy import deepcopy
+from dataclasses import replace
+
+import pytest
+
+from openultrasast.hunt.accept import FoldResult, Rule, decide, record_rejection
+from openultrasast.hunt.artifact import RunArtifact
+from openultrasast.hunt.propose import Candidate
+
+
+def estimate(diff=0.2):
+    """A synthetic paired estimate with counts and a finite interval."""
+    return {"diff": diff, "ci": [diff - 0.1, diff + 0.1], "units": 20, "groups": 4}
+
+
+@pytest.fixture
+def candidate(tmp_path):
+    """A single instruction edit, with its baseline available for comparison."""
+    base = {"hunt": {"signature": {"instructions": "old", "fields": []}, "demos": []}}
+    state = deepcopy(base)
+    state["hunt"]["signature"]["instructions"] = "new"
+    return Candidate(
+        "candidate",
+        "instruction",
+        {"instruction": {"hunt": "new"}},
+        {"recall": 0.7, "ci": [0.5, 0.9]},
+        state,
+        "current",
+        base,
+        (),
+        (),
+        str(tmp_path),
+    )
+
+
+@pytest.fixture
+def results():
+    """Two scopes containing pooled and family differences."""
+    return (
+        {family: FoldResult("compile", family, estimate(), estimate(0)) for family in ("pooled", "injection")},
+        {family: FoldResult("holdout", family, estimate(), estimate(0)) for family in ("pooled", "injection")},
+    )
+
+
+def rule(**kwargs):
+    """A fixed rule with no permitted family regression by default."""
+    return Rule(**({"tolerance": 0.01, "regression_budget": {"default": 0, "injection": -0.1}, "instruction_cap": 100} | kwargs))
+
+
+def difference(results, scope, family, value):
+    """Change one measured recall difference."""
+    results[scope][family] = replace(results[scope][family], recall=estimate(value))
+
+
+@pytest.mark.parametrize("scope,reason", [(0, "compile_recall_negative"), (1, "holdout_recall_negative")])
+def test_negative_pooled_difference_rejects(candidate, results, scope, reason):
+    """Either negative pooled result defeats an otherwise acceptable proposal."""
+    assert decide("current", candidate, *results, rule()).accepted
+    difference(results, scope, "pooled", -0.01)
+    decision = decide("current", candidate, *results, rule())
+    assert not decision.accepted and reason in decision.reasons
+
+
+def test_both_zero_rejects(candidate, results):
+    """Zero gain in both scopes is insufficient and is a held-out tie."""
+    difference(results, 0, "pooled", 0)
+    difference(results, 1, "pooled", 0)
+    decision = decide("current", candidate, *results, rule())
+    assert not decision.accepted
+    assert {"no_positive_recall_difference", "holdout_tie"} <= set(decision.reasons)
+
+
+def test_compile_zero_holdout_positive_accepts(candidate, results):
+    """A positive held-out gain can accept when compile recall is unchanged."""
+    difference(results, 0, "pooled", 0)
+    assert decide("current", candidate, *results, rule()).accepted
+
+
+def test_heldout_tolerance_rejects_even_with_compile_gain(candidate, results):
+    """The tolerance boundary is inclusive and applies to held-out recall."""
+    difference(results, 1, "pooled", 0.01)
+    assert "holdout_tie" in decide("current", candidate, *results, rule()).reasons
+    difference(results, 1, "pooled", 0.0101)
+    assert decide("current", candidate, *results, rule()).accepted
+
+
+def test_family_budget_overrides_pooled_gain(candidate, results):
+    """A family at its allowed floor passes; below it rejects."""
+    difference(results, 1, "injection", -0.1)
+    assert decide("current", candidate, *results, rule()).accepted
+    difference(results, 1, "injection", -0.101)
+    assert "family_regression:injection" in decide("current", candidate, *results, rule()).reasons
+
+
+def test_unlisted_family_uses_default_budget(candidate, results):
+    """Families without an override still have a regression floor."""
+    for scope, name in enumerate(("compile", "holdout")):
+        results[scope]["path"] = FoldResult(name, "path", estimate(-0.01), estimate(0))
+    assert "family_regression:path" in decide("current", candidate, *results, rule()).reasons
+    assert decide("current", candidate, *results, rule(regression_budget={"default": -0.02})).accepted
+
+
+def test_instruction_cap_uses_saved_instruction(candidate, results):
+    """The state is authoritative even when the reported change is short."""
+    assert decide("current", candidate, *results, rule(instruction_cap=3)).accepted
+    decision = decide("current", candidate, *results, rule(instruction_cap=2))
+    assert not decision.accepted and "instruction_cap" in decision.reasons
+
+
+@pytest.mark.parametrize("scope,family,metric", [(0, "pooled", "recall"), (1, "injection", "recall"), (1, "pooled", "suspicions_per_push")])
+def test_missing_interval_rejects(candidate, results, scope, family, metric):
+    """Neither a family nor a suspicion point estimate can stand in for an interval."""
+    result = results[scope][family]
+    results[scope][family] = replace(result, **{metric: {**getattr(result, metric), "ci": None}})
+    assert not decide("current", candidate, *results, rule()).accepted
+    assert any(r.startswith("missing_interval:") for r in decide("current", candidate, *results, rule()).reasons)
+
+
+def test_multiple_changes_reject_even_when_kind_claims_instruction(candidate, results):
+    """Saved demos reveal an undeclared second edit."""
+    state = deepcopy(candidate.state)
+    state["hunt"]["demos"] = [{"findings": []}]
+    decision = decide("current", replace(candidate, state=state), *results, rule())
+    assert not decision.accepted and "multiple_or_invalid_changes" in decision.reasons
+
+
+def test_comparison_is_saved_and_fits_existing_artifact(candidate, results, tmp_path):
+    """Both IDs, scopes and the rule survive an artifact round trip."""
+    decision = decide("current", candidate, *results, rule())
+    assert decision.comparison["current_id"] == "current"
+    assert decision.comparison["candidate_id"] == "candidate"
+    assert decision.comparison["compile_results"]["pooled"]["recall"]["diff"] == 0.2
+    assert decision.comparison["holdout_results"]["injection"]["recall"]["ci"]
+    assert decision.comparison["rule"]["version"] == "accept-v1"
+    assert json.loads((tmp_path / "comparison.json").read_text()) == decision.comparison
+    artifact = RunArtifact("candidate", 1, (), {}, [], {}, {}, "fixture")
+    decision.record(tmp_path, artifact=artifact)
+    assert artifact.configurations == [{"comparison": decision.comparison}]
+    path = artifact.write(tmp_path)
+    assert RunArtifact.read(path).configurations == artifact.configurations
+
+
+def test_rejection_omits_heldout_numbers(candidate, results, tmp_path):
+    """The rejection journal discloses reasons and compile measurements only."""
+    difference(results, 1, "pooled", -0.987654)
+    decision = decide("current", candidate, *results, rule())
+    record_rejection(tmp_path, candidate, decision)
+    record_rejection(tmp_path, candidate, decision)
+    rows = [json.loads(line) for line in (tmp_path / "rejected.jsonl").read_text().splitlines()]
+    assert len(rows) == 2 and rows[0]["reasons"] == decision.reasons
+    assert rows[0]["compile_score"] == candidate.compile_score
+    assert "-0.987654" not in json.dumps(rows)
+    assert set(rows[0]) == {"program_id", "kind", "change", "compile_score", "reasons"}
+
+
+def test_missing_family_and_wrong_scope_cannot_accept(candidate, results):
+    """Incomplete or mislabelled comparisons are not usable evidence."""
+    del results[1]["injection"]
+    assert not decide("current", candidate, *results, rule()).accepted
+    results[1]["pooled"] = replace(results[1]["pooled"], scope="compile")
+    assert not decide("current", candidate, *results, rule()).accepted

tests/test_hunt_propose.py

@@ -0,0 +1,213 @@
+"""Compile-only proposals on six changes and a scripted model."""
+
+import json
+from copy import deepcopy
+
+import pytest
+
+dspy = pytest.importorskip("dspy")
+pytest.importorskip("statsmodels")
+from dspy.utils import DummyLM  # noqa: E402
+
+from openultrasast.hunt import program as hp  # noqa: E402
+from openultrasast.hunt.propose import OPTIMISERS, Round, compile_examples, propose, visible_to_proposer  # noqa: E402
+
+
+@pytest.fixture
+def population(tmp_path):
+    """Read real source bytes for three repositories with two changes each."""
+    changes, rows = [], {}
+    source = tmp_path / "app.py"
+    source.write_text("def run():\n    return eval(input())\n")
+    data = source.read_bytes()
+    print(f"fixture app.py: {len(data)} bytes read")
+    assert len(data) > 0
+    for repo in range(3):
+        for item in range(2):
+            change = dict(
+                id=f"c{repo}-{item}",
+                repo=f"repo-{repo}",
+                fold=repo,
+                kind="vulnerability" if item == 0 else "ordinary",
+                family="injection",
+                path="app.py",
+                files=[{"path": "app.py", "head_lines": [[2, 2]]}],
+                advisory="PRIVATE-ADVISORY",
+                verification="PRIVATE-VERIFICATION",
+            )
+            changes.append(change)
+            rows[change["id"]] = [
+                {
+                    "path": "app.py",
+                    "fold": repo,
+                    "label": int(item == 0),
+                    "feature.source.function": f"run{repo}{item}",
+                    "feature.source.span": [1, 2],
+                    "feature.diff.hunks": [[2, 2]],
+                    "verification": "PRIVATE-VERIFICATION",
+                }
+            ]
+
+    def render(change, selected):
+        """Bind source text to the shipped renderer."""
+        return hp.render_rows(change, selected, {"app.py": data.decode().splitlines()})
+
+    return changes, rows, render
+
+
+def instruction(program, examples, metric):
+    """Change only the instruction using the same callable interface as an optimiser."""
+    assert callable(metric)
+    program.hunt.signature = program.hunt.signature.with_instructions("Check the changed interpreter call.")
+    return program
+
+
+def run_round(population, tmp_path, optimiser=instruction, kind="instruction"):
+    """Run a fixed baseline and candidate sequence without network access."""
+    changes, rows, render = population
+    examples = compile_examples(changes, rows, compile_folds=[0, 1], render=render)
+    program = hp.HuntProgram()
+    round = Round(kind, hp.program_id(program), [0, 1], 13, "callable", {"usd": 1, "calls": 100})
+    finding = {"path": "app.py", "line": 2, "family": "injection", "cited_lines": [2]}
+    lm = DummyLM([{"findings": "[]"}] * 4 + [{"findings": json.dumps([finding])}] * 4)
+    with dspy.context(lm=lm, adapter=dspy.ChatAdapter()):
+        candidate = propose(round, program, examples, optimiser=optimiser, out_dir=tmp_path)
+    return candidate, program, examples
+
+
+def test_instruction_scores_only_compile_changes_and_preserves_base(population, tmp_path):
+    """The optimiser and provenance see four compile changes, never the two held out."""
+
+    def checked(program, examples, metric):
+        assert [e.change_id for e in examples] == ["c0-0", "c0-1", "c1-0", "c1-1"]
+        for example in examples:
+            assert "change" not in example.inputs()
+            assert "PRIVATE" not in json.dumps(example.toDict())
+        return instruction(program, examples, metric)
+
+    candidate, base, examples = run_round(population, tmp_path, checked)
+    assert candidate.program_id != hp.program_id(base)
+    assert base.hunt.signature.instructions == hp.BASELINE_INSTRUCTION
+    assert candidate.compile_score["recall"] == 1
+    assert candidate.compile_score["ci"] is not None
+    assert candidate.compile_score["positive_units"] == 2
+    assert candidate.compile_score["paired"]["recall"]["diff"] == 1
+    assert candidate.compile_score["paired"]["recall"]["groups"] == 2
+    provenance = json.loads((tmp_path / "proposer_inputs.json").read_text())
+    assert provenance == [{"change_id": e.change_id, "fold": e.fold} for e in examples]
+    assert "c2-" not in json.dumps(provenance)
+    saved = hp.HuntProgram()
+    saved.load_state(candidate.state)
+    assert hp.program_id(saved, mask=candidate.mask) == candidate.program_id
+
+
+def test_two_surfaces_are_refused(population, tmp_path):
+    """Instruction and demo edits cannot enter the same round."""
+
+    def two(program, examples, metric):
+        instruction(program, examples, metric)
+        program.hunt.demos = [dspy.Example(**examples[0].inputs(), findings=[])]
+        return program
+
+    with pytest.raises(ValueError, match="one"):
+        run_round(population, tmp_path, two)
+    assert not (tmp_path / "candidates.jsonl").exists()
+
+
+def test_history_contains_all_candidates_but_no_holdout(population, tmp_path):
+    """Acceptance records in the same directory cannot leak through the history reader."""
+    candidate, _, _ = run_round(population, tmp_path)
+
+    def other_instruction(program, examples, metric):
+        """Produce a distinct second candidate for the same compile population."""
+        program.hunt.signature = program.hunt.signature.with_instructions("Follow the changed value into the interpreter.")
+        return program
+
+    second, _, _ = run_round(population, tmp_path, other_instruction)
+    (tmp_path / "comparison.json").write_text('{"holdout": 987.654}')
+    (tmp_path / "rejected.jsonl").write_text(json.dumps({"program_id": second.program_id, "holdout": 987.654}) + "\n")
+    history = visible_to_proposer(tmp_path)
+    assert [record["program_id"] for record in history] == [candidate.program_id, second.program_id]
+    assert history[0]["compile_score"]["recall"] == 1
+    assert "holdout" not in json.dumps(history) and "987.654" not in json.dumps(history)
+
+
+def test_demo_and_mask_rounds(population, tmp_path):
+    """Single demo and mask changes retain their actual provenance."""
+
+    def demos(program, examples, metric):
+        program.hunt.demos = [dspy.Example(**examples[0].inputs(), findings=[])]
+        return program
+
+    demo, _, _ = run_round(population, tmp_path / "demos", demos, "demos")
+    assert demo.change == {"demos": {"count": 1, "change_ids": ["c0-0"]}}
+    mask, _, _ = run_round(population, tmp_path / "mask", lambda p, e, m: hp.HuntProgram(mask=["roles"]), "mask")
+    assert mask.change == {"mask": ["roles"]}
+
+
+def test_examples_are_not_an_editable_answer_key(population, tmp_path):
+    """An optimiser's mutation of an example cannot change the paired scoring population."""
+
+    def mutate(program, examples, metric):
+        examples[0].change["kind"] = "ordinary"
+        return instruction(program, examples, metric)
+
+    candidate, _, examples = run_round(population, tmp_path, mutate)
+    assert candidate.compile_score["positive_units"] == 2
+    assert examples[0].change["kind"] == "vulnerability"
+
+
+def test_invalid_round_and_cross_fold_repository_are_refused(population):
+    """A round has one kind, and a repository cannot straddle compile and holdout."""
+    with pytest.raises(ValueError, match="kind"):
+        Round(("instruction", "demos"), "base", [0], 0, "callable", {"usd": 1, "calls": 10})
+    changes, rows, render = deepcopy(population)
+    changes[-1]["repo"] = changes[0]["repo"]
+    with pytest.raises(ValueError, match="repository"):
+        compile_examples(changes, rows, compile_folds=[0, 1], render=render)
+
+
+def test_registry_resolves_dspy_classes():
+    """Registry lookups load the real DSPy optimiser classes."""
+    assert OPTIMISERS["gepa"] is dspy.GEPA
+    assert OPTIMISERS["miprov2"] is dspy.MIPROv2
+    assert OPTIMISERS["bootstrap"] is dspy.BootstrapFewShot
+
+
+@pytest.mark.parametrize("name", ["gepa", "miprov2", "bootstrap", "configured-miprov2"])
+def test_named_optimisers_construct_and_receive_only_compile_examples(population, tmp_path, monkeypatch, name):
+    """Exercise real constructors and the compile call without running a model search."""
+    changes, rows, render = population
+    examples = compile_examples(changes, rows, compile_folds=[0, 1], render=render)
+    base = hp.HuntProgram()
+    called = []
+
+    def compile(self, student, *, trainset, **kwargs):
+        """Inspect the real optimiser and supply a deterministic compiled result."""
+        called.append(type(self))
+        assert {e.fold for e in trainset} == {0, 1}
+        assert all("change" not in e.inputs() for e in trainset)
+        if name == "gepa":
+            feedback = self.metric_fn(trainset[0], dspy.Prediction(findings=[]))
+            assert feedback.score == 0 and "missed" in feedback.feedback
+        if name.endswith("miprov2"):
+            assert kwargs == {"data_aware_proposer": False}
+        return instruction(student, trainset, lambda *args: 0)
+
+    cls = OPTIMISERS[name.removeprefix("configured-")]
+    monkeypatch.setattr(cls, "compile", compile)
+    round = Round("instruction", hp.program_id(base), [0, 1], 13, name, {"usd": 1, "calls": 100})
+    with dspy.context(lm=DummyLM([{"findings": "[]"}] * 8), adapter=dspy.ChatAdapter()):
+        optimiser = cls(metric=lambda *args: 0) if name.startswith("configured-") else name
+        candidate = propose(round, base, examples, optimiser=optimiser, out_dir=tmp_path)
+    assert called == [cls] and candidate.compile_score["ci"]
+
+
+def test_gepa_without_model_fails_clearly(population, tmp_path):
+    """A missing reflection model is diagnosed before an optimiser is launched."""
+    changes, rows, render = population
+    examples = compile_examples(changes, rows, compile_folds=[0, 1], render=render)
+    base = hp.HuntProgram()
+    round = Round("instruction", hp.program_id(base), [0, 1], 13, "gepa", {"usd": 1, "calls": 100})
+    with dspy.context(lm=None), pytest.raises(ValueError, match="reflection model"):
+        propose(round, base, examples, optimiser="gepa", out_dir=tmp_path)