mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-19 13:10:03 +00:00
feat(hunt): proposals and the acceptance rule (5.1, 5.2)
propose.py: one change per round (instruction, demonstrations or one mask), examples built from compile folds only with a provenance record of the change ids used, a registry for GEPA, MIPROv2 and BootstrapFewShot (MIPROv2 without the data-aware proposer so labels are never serialised), compile-fold scoring with Wilson and the repository-paired bootstrap, and a compile-only history the next round may read. accept.py: the rule as independent clauses (both folds non-negative with one strictly positive, per-family regression budget on held-out recall, tie tolerance, no acceptance without an interval, instruction cap, one change only), the comparison recorded in the candidate artifact's configurations, rejections logged without held-out numbers. 48 fixture tests. Known gap for live rounds: the optimisers and propose._score call the program directly, so a tool-using program must be scored through the runner with its tools bound per checkout (6.1). Implemented by Codex (gpt-6-astra), gated here. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
47d0f0fac1
commit
bacd7a1e3f
@@ -0,0 +1,180 @@
|
||||
"""Accept a proposal only when separate compile and held-out comparisons support it."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import asdict, dataclass, is_dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..stats import Estimate
|
||||
from .artifact import RunArtifact
|
||||
from .propose import Candidate, _append, _write, changed_surfaces, proposer_record
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Rule:
|
||||
"""The acceptance rule fixed before measurement.
|
||||
|
||||
Regression budgets are signed recall differences: -0.02 permits a loss of
|
||||
two percentage points. The 'default' entry applies to unlisted families;
|
||||
when omitted it is zero. The held-out tie band includes both endpoints.
|
||||
"""
|
||||
|
||||
tolerance: float
|
||||
regression_budget: Mapping[str, float]
|
||||
instruction_cap: int
|
||||
version: str = "accept-v1"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Reject malformed or unsupported rules before using any measurements."""
|
||||
if self.version != "accept-v1":
|
||||
raise ValueError("unsupported acceptance rule version")
|
||||
if not math.isfinite(self.tolerance) or self.tolerance < 0:
|
||||
raise ValueError("tolerance must be finite and non-negative")
|
||||
if type(self.instruction_cap) is not int or self.instruction_cap < 0:
|
||||
raise ValueError("instruction cap must be a non-negative integer")
|
||||
if any(not math.isfinite(v) or not -1 <= v <= 0 for v in self.regression_budget.values()):
|
||||
raise ValueError("regression budgets must be signed differences between -1 and zero")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FoldResult:
|
||||
"""Paired candidate-minus-current differences for one scope and one family.
|
||||
|
||||
Use family='pooled' for the aggregate and scope='compile' or 'holdout'. Each
|
||||
metric accepts stats.Estimate or a record containing diff, ci, units and
|
||||
groups. Missing intervals remain explicit so decide can reject them.
|
||||
"""
|
||||
|
||||
scope: str
|
||||
family: str
|
||||
recall: Estimate | Mapping[str, Any]
|
||||
suspicions_per_push: Estimate | Mapping[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Decision:
|
||||
"""The verdict, each failed clause, and the full private comparison."""
|
||||
|
||||
accepted: bool
|
||||
reasons: list[str]
|
||||
comparison: dict[str, Any]
|
||||
|
||||
def record(self, out_dir: Path | str, *, artifact: RunArtifact | None = None) -> None:
|
||||
"""Save comparison.json and optionally attach it to the candidate's artifact.
|
||||
|
||||
The existing configurations field carries the full comparison. Call
|
||||
artifact.write afterwards to persist its new content address.
|
||||
"""
|
||||
if artifact is not None and artifact.program_id != self.comparison["candidate_id"]:
|
||||
raise ValueError("comparison belongs to a different program artifact")
|
||||
_write(Path(out_dir) / "comparison.json", self.comparison)
|
||||
if artifact is not None:
|
||||
record = {"comparison": self.comparison}
|
||||
if record not in artifact.configurations:
|
||||
artifact.configurations.append(record)
|
||||
|
||||
|
||||
def _record(value: Any) -> dict[str, Any]:
|
||||
"""Convert dataclass estimates and ordinary mappings to JSON records."""
|
||||
return asdict(value) if is_dataclass(value) else dict(value)
|
||||
|
||||
|
||||
def _valid_number(value: Any) -> bool:
|
||||
"""A missing or non-finite point estimate cannot support acceptance."""
|
||||
return isinstance(value, int | float) and not isinstance(value, bool) and math.isfinite(value)
|
||||
|
||||
|
||||
def _interval_present(estimate: Mapping[str, Any]) -> bool:
|
||||
"""Require an ordered finite interval and real unit and group counts."""
|
||||
ci = estimate.get("ci")
|
||||
return (
|
||||
isinstance(ci, tuple | list)
|
||||
and len(ci) == 2
|
||||
and all(_valid_number(v) for v in ci)
|
||||
and ci[0] <= ci[1]
|
||||
and _valid_number(estimate.get("diff"))
|
||||
and type(estimate.get("units")) is int
|
||||
and type(estimate.get("groups")) is int
|
||||
and 0 < estimate["groups"] <= estimate["units"]
|
||||
)
|
||||
|
||||
|
||||
def decide(
|
||||
current_id: str, candidate: Candidate, compile_results: Mapping[str, FoldResult], holdout_results: Mapping[str, FoldResult], rule: Rule
|
||||
) -> Decision:
|
||||
"""Apply every clause independently and save the comparison beside the proposal.
|
||||
|
||||
A zero compile difference and positive held-out difference may accept. A
|
||||
positive compile difference with a held-out tie rejects. Confidence
|
||||
intervals must exist; they need not exclude zero under this rule.
|
||||
"""
|
||||
rule.__post_init__()
|
||||
reasons = []
|
||||
if candidate.base_program_id != current_id:
|
||||
reasons.append("current_program_mismatch")
|
||||
try:
|
||||
surfaces = changed_surfaces(candidate.base_state, candidate.state, candidate.base_mask, candidate.mask)
|
||||
if surfaces != {candidate.kind} or set(candidate.change) != {candidate.kind}:
|
||||
reasons.append("multiple_or_invalid_changes")
|
||||
except ValueError:
|
||||
reasons.append("multiple_or_invalid_changes")
|
||||
instructions = [state.get("signature", {}).get("instructions", "") for state in candidate.state.values()]
|
||||
if any(not isinstance(text, str) or len(text) > rule.instruction_cap for text in instructions):
|
||||
reasons.append("instruction_cap")
|
||||
compiled, heldout = {}, {}
|
||||
for scope, source, target in (("compile", compile_results, compiled), ("holdout", holdout_results, heldout)):
|
||||
for family, result in source.items():
|
||||
record = _record(result)
|
||||
if record.get("scope") != scope or record.get("family") != family:
|
||||
reasons.append(f"invalid_scope:{scope}:{family}")
|
||||
target[family] = record
|
||||
for metric in ("recall", "suspicions_per_push"):
|
||||
estimate = _record(record.get(metric, {}))
|
||||
record[metric] = estimate
|
||||
if not _interval_present(estimate):
|
||||
reasons.append(f"missing_interval:{scope}:{family}:{metric}")
|
||||
if "pooled" not in source:
|
||||
reasons.append(f"missing_pooled:{scope}")
|
||||
if set(compiled) != set(heldout):
|
||||
reasons.append("missing_family_comparison")
|
||||
compile_diff = compiled.get("pooled", {}).get("recall", {}).get("diff")
|
||||
holdout_diff = heldout.get("pooled", {}).get("recall", {}).get("diff")
|
||||
if _valid_number(compile_diff) and compile_diff < 0:
|
||||
reasons.append("compile_recall_negative")
|
||||
if _valid_number(holdout_diff) and holdout_diff < 0:
|
||||
reasons.append("holdout_recall_negative")
|
||||
if not any(_valid_number(value) and value > 0 for value in (compile_diff, holdout_diff)):
|
||||
reasons.append("no_positive_recall_difference")
|
||||
if _valid_number(holdout_diff) and abs(holdout_diff) <= rule.tolerance:
|
||||
reasons.append("holdout_tie")
|
||||
for family, result in heldout.items():
|
||||
if family == "pooled":
|
||||
continue
|
||||
diff = result["recall"].get("diff")
|
||||
floor = rule.regression_budget.get(family, rule.regression_budget.get("default", 0))
|
||||
if _valid_number(diff) and diff < floor:
|
||||
reasons.append(f"family_regression:{family}")
|
||||
comparison = {
|
||||
"current_id": current_id,
|
||||
"candidate_id": candidate.program_id,
|
||||
"compile_results": compiled,
|
||||
"holdout_results": heldout,
|
||||
"rule": asdict(rule),
|
||||
"accepted": not reasons,
|
||||
"reasons": reasons,
|
||||
}
|
||||
decision = Decision(not reasons, reasons, comparison)
|
||||
decision.record(candidate.out_dir)
|
||||
return decision
|
||||
|
||||
|
||||
def record_rejection(out_dir: Path | str, candidate: Candidate, decision: Decision) -> None:
|
||||
"""Append the compile score and reason codes, never the private comparison."""
|
||||
if decision.accepted:
|
||||
raise ValueError("cannot record an accepted candidate as rejected")
|
||||
if decision.comparison["candidate_id"] != candidate.program_id:
|
||||
raise ValueError("decision belongs to a different candidate")
|
||||
_append(Path(out_dir) / "rejected.jsonl", {**proposer_record(candidate), "reasons": decision.reasons})
|
||||
Reference in New Issue
Block a user