mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
feat(hunt): proposals and the acceptance rule (5.1, 5.2)
propose.py: one change per round (instruction, demonstrations or one mask), examples built from compile folds only with a provenance record of the change ids used, a registry for GEPA, MIPROv2 and BootstrapFewShot (MIPROv2 without the data-aware proposer so labels are never serialised), compile-fold scoring with Wilson and the repository-paired bootstrap, and a compile-only history the next round may read. accept.py: the rule as independent clauses (both folds non-negative with one strictly positive, per-family regression budget on held-out recall, tie tolerance, no acceptance without an interval, instruction cap, one change only), the comparison recorded in the candidate artifact's configurations, rejections logged without held-out numbers. 48 fixture tests. Known gap for live rounds: the optimisers and propose._score call the program directly, so a tool-using program must be scored through the runner with its tools bound per checkout (6.1). Implemented by Codex (gpt-6-astra), gated here. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
47d0f0fac1
commit
bacd7a1e3f
@@ -0,0 +1,164 @@
|
||||
"""Each acceptance clause has an independent numerical counterexample."""
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.hunt.accept import FoldResult, Rule, decide, record_rejection
|
||||
from openultrasast.hunt.artifact import RunArtifact
|
||||
from openultrasast.hunt.propose import Candidate
|
||||
|
||||
|
||||
def estimate(diff=0.2):
|
||||
"""A synthetic paired estimate with counts and a finite interval."""
|
||||
return {"diff": diff, "ci": [diff - 0.1, diff + 0.1], "units": 20, "groups": 4}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def candidate(tmp_path):
|
||||
"""A single instruction edit, with its baseline available for comparison."""
|
||||
base = {"hunt": {"signature": {"instructions": "old", "fields": []}, "demos": []}}
|
||||
state = deepcopy(base)
|
||||
state["hunt"]["signature"]["instructions"] = "new"
|
||||
return Candidate(
|
||||
"candidate",
|
||||
"instruction",
|
||||
{"instruction": {"hunt": "new"}},
|
||||
{"recall": 0.7, "ci": [0.5, 0.9]},
|
||||
state,
|
||||
"current",
|
||||
base,
|
||||
(),
|
||||
(),
|
||||
str(tmp_path),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def results():
|
||||
"""Two scopes containing pooled and family differences."""
|
||||
return (
|
||||
{family: FoldResult("compile", family, estimate(), estimate(0)) for family in ("pooled", "injection")},
|
||||
{family: FoldResult("holdout", family, estimate(), estimate(0)) for family in ("pooled", "injection")},
|
||||
)
|
||||
|
||||
|
||||
def rule(**kwargs):
|
||||
"""A fixed rule with no permitted family regression by default."""
|
||||
return Rule(**({"tolerance": 0.01, "regression_budget": {"default": 0, "injection": -0.1}, "instruction_cap": 100} | kwargs))
|
||||
|
||||
|
||||
def difference(results, scope, family, value):
|
||||
"""Change one measured recall difference."""
|
||||
results[scope][family] = replace(results[scope][family], recall=estimate(value))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scope,reason", [(0, "compile_recall_negative"), (1, "holdout_recall_negative")])
|
||||
def test_negative_pooled_difference_rejects(candidate, results, scope, reason):
|
||||
"""Either negative pooled result defeats an otherwise acceptable proposal."""
|
||||
assert decide("current", candidate, *results, rule()).accepted
|
||||
difference(results, scope, "pooled", -0.01)
|
||||
decision = decide("current", candidate, *results, rule())
|
||||
assert not decision.accepted and reason in decision.reasons
|
||||
|
||||
|
||||
def test_both_zero_rejects(candidate, results):
|
||||
"""Zero gain in both scopes is insufficient and is a held-out tie."""
|
||||
difference(results, 0, "pooled", 0)
|
||||
difference(results, 1, "pooled", 0)
|
||||
decision = decide("current", candidate, *results, rule())
|
||||
assert not decision.accepted
|
||||
assert {"no_positive_recall_difference", "holdout_tie"} <= set(decision.reasons)
|
||||
|
||||
|
||||
def test_compile_zero_holdout_positive_accepts(candidate, results):
|
||||
"""A positive held-out gain can accept when compile recall is unchanged."""
|
||||
difference(results, 0, "pooled", 0)
|
||||
assert decide("current", candidate, *results, rule()).accepted
|
||||
|
||||
|
||||
def test_heldout_tolerance_rejects_even_with_compile_gain(candidate, results):
|
||||
"""The tolerance boundary is inclusive and applies to held-out recall."""
|
||||
difference(results, 1, "pooled", 0.01)
|
||||
assert "holdout_tie" in decide("current", candidate, *results, rule()).reasons
|
||||
difference(results, 1, "pooled", 0.0101)
|
||||
assert decide("current", candidate, *results, rule()).accepted
|
||||
|
||||
|
||||
def test_family_budget_overrides_pooled_gain(candidate, results):
|
||||
"""A family at its allowed floor passes; below it rejects."""
|
||||
difference(results, 1, "injection", -0.1)
|
||||
assert decide("current", candidate, *results, rule()).accepted
|
||||
difference(results, 1, "injection", -0.101)
|
||||
assert "family_regression:injection" in decide("current", candidate, *results, rule()).reasons
|
||||
|
||||
|
||||
def test_unlisted_family_uses_default_budget(candidate, results):
|
||||
"""Families without an override still have a regression floor."""
|
||||
for scope, name in enumerate(("compile", "holdout")):
|
||||
results[scope]["path"] = FoldResult(name, "path", estimate(-0.01), estimate(0))
|
||||
assert "family_regression:path" in decide("current", candidate, *results, rule()).reasons
|
||||
assert decide("current", candidate, *results, rule(regression_budget={"default": -0.02})).accepted
|
||||
|
||||
|
||||
def test_instruction_cap_uses_saved_instruction(candidate, results):
|
||||
"""The state is authoritative even when the reported change is short."""
|
||||
assert decide("current", candidate, *results, rule(instruction_cap=3)).accepted
|
||||
decision = decide("current", candidate, *results, rule(instruction_cap=2))
|
||||
assert not decision.accepted and "instruction_cap" in decision.reasons
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scope,family,metric", [(0, "pooled", "recall"), (1, "injection", "recall"), (1, "pooled", "suspicions_per_push")])
|
||||
def test_missing_interval_rejects(candidate, results, scope, family, metric):
|
||||
"""Neither a family nor a suspicion point estimate can stand in for an interval."""
|
||||
result = results[scope][family]
|
||||
results[scope][family] = replace(result, **{metric: {**getattr(result, metric), "ci": None}})
|
||||
assert not decide("current", candidate, *results, rule()).accepted
|
||||
assert any(r.startswith("missing_interval:") for r in decide("current", candidate, *results, rule()).reasons)
|
||||
|
||||
|
||||
def test_multiple_changes_reject_even_when_kind_claims_instruction(candidate, results):
|
||||
"""Saved demos reveal an undeclared second edit."""
|
||||
state = deepcopy(candidate.state)
|
||||
state["hunt"]["demos"] = [{"findings": []}]
|
||||
decision = decide("current", replace(candidate, state=state), *results, rule())
|
||||
assert not decision.accepted and "multiple_or_invalid_changes" in decision.reasons
|
||||
|
||||
|
||||
def test_comparison_is_saved_and_fits_existing_artifact(candidate, results, tmp_path):
|
||||
"""Both IDs, scopes and the rule survive an artifact round trip."""
|
||||
decision = decide("current", candidate, *results, rule())
|
||||
assert decision.comparison["current_id"] == "current"
|
||||
assert decision.comparison["candidate_id"] == "candidate"
|
||||
assert decision.comparison["compile_results"]["pooled"]["recall"]["diff"] == 0.2
|
||||
assert decision.comparison["holdout_results"]["injection"]["recall"]["ci"]
|
||||
assert decision.comparison["rule"]["version"] == "accept-v1"
|
||||
assert json.loads((tmp_path / "comparison.json").read_text()) == decision.comparison
|
||||
artifact = RunArtifact("candidate", 1, (), {}, [], {}, {}, "fixture")
|
||||
decision.record(tmp_path, artifact=artifact)
|
||||
assert artifact.configurations == [{"comparison": decision.comparison}]
|
||||
path = artifact.write(tmp_path)
|
||||
assert RunArtifact.read(path).configurations == artifact.configurations
|
||||
|
||||
|
||||
def test_rejection_omits_heldout_numbers(candidate, results, tmp_path):
|
||||
"""The rejection journal discloses reasons and compile measurements only."""
|
||||
difference(results, 1, "pooled", -0.987654)
|
||||
decision = decide("current", candidate, *results, rule())
|
||||
record_rejection(tmp_path, candidate, decision)
|
||||
record_rejection(tmp_path, candidate, decision)
|
||||
rows = [json.loads(line) for line in (tmp_path / "rejected.jsonl").read_text().splitlines()]
|
||||
assert len(rows) == 2 and rows[0]["reasons"] == decision.reasons
|
||||
assert rows[0]["compile_score"] == candidate.compile_score
|
||||
assert "-0.987654" not in json.dumps(rows)
|
||||
assert set(rows[0]) == {"program_id", "kind", "change", "compile_score", "reasons"}
|
||||
|
||||
|
||||
def test_missing_family_and_wrong_scope_cannot_accept(candidate, results):
|
||||
"""Incomplete or mislabelled comparisons are not usable evidence."""
|
||||
del results[1]["injection"]
|
||||
assert not decide("current", candidate, *results, rule()).accepted
|
||||
results[1]["pooled"] = replace(results[1]["pooled"], scope="compile")
|
||||
assert not decide("current", candidate, *results, rule()).accepted
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Compile-only proposals on six changes and a scripted model."""
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
|
||||
dspy = pytest.importorskip("dspy")
|
||||
pytest.importorskip("statsmodels")
|
||||
from dspy.utils import DummyLM # noqa: E402
|
||||
|
||||
from openultrasast.hunt import program as hp # noqa: E402
|
||||
from openultrasast.hunt.propose import OPTIMISERS, Round, compile_examples, propose, visible_to_proposer # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def population(tmp_path):
|
||||
"""Read real source bytes for three repositories with two changes each."""
|
||||
changes, rows = [], {}
|
||||
source = tmp_path / "app.py"
|
||||
source.write_text("def run():\n return eval(input())\n")
|
||||
data = source.read_bytes()
|
||||
print(f"fixture app.py: {len(data)} bytes read")
|
||||
assert len(data) > 0
|
||||
for repo in range(3):
|
||||
for item in range(2):
|
||||
change = dict(
|
||||
id=f"c{repo}-{item}",
|
||||
repo=f"repo-{repo}",
|
||||
fold=repo,
|
||||
kind="vulnerability" if item == 0 else "ordinary",
|
||||
family="injection",
|
||||
path="app.py",
|
||||
files=[{"path": "app.py", "head_lines": [[2, 2]]}],
|
||||
advisory="PRIVATE-ADVISORY",
|
||||
verification="PRIVATE-VERIFICATION",
|
||||
)
|
||||
changes.append(change)
|
||||
rows[change["id"]] = [
|
||||
{
|
||||
"path": "app.py",
|
||||
"fold": repo,
|
||||
"label": int(item == 0),
|
||||
"feature.source.function": f"run{repo}{item}",
|
||||
"feature.source.span": [1, 2],
|
||||
"feature.diff.hunks": [[2, 2]],
|
||||
"verification": "PRIVATE-VERIFICATION",
|
||||
}
|
||||
]
|
||||
|
||||
def render(change, selected):
|
||||
"""Bind source text to the shipped renderer."""
|
||||
return hp.render_rows(change, selected, {"app.py": data.decode().splitlines()})
|
||||
|
||||
return changes, rows, render
|
||||
|
||||
|
||||
def instruction(program, examples, metric):
|
||||
"""Change only the instruction using the same callable interface as an optimiser."""
|
||||
assert callable(metric)
|
||||
program.hunt.signature = program.hunt.signature.with_instructions("Check the changed interpreter call.")
|
||||
return program
|
||||
|
||||
|
||||
def run_round(population, tmp_path, optimiser=instruction, kind="instruction"):
|
||||
"""Run a fixed baseline and candidate sequence without network access."""
|
||||
changes, rows, render = population
|
||||
examples = compile_examples(changes, rows, compile_folds=[0, 1], render=render)
|
||||
program = hp.HuntProgram()
|
||||
round = Round(kind, hp.program_id(program), [0, 1], 13, "callable", {"usd": 1, "calls": 100})
|
||||
finding = {"path": "app.py", "line": 2, "family": "injection", "cited_lines": [2]}
|
||||
lm = DummyLM([{"findings": "[]"}] * 4 + [{"findings": json.dumps([finding])}] * 4)
|
||||
with dspy.context(lm=lm, adapter=dspy.ChatAdapter()):
|
||||
candidate = propose(round, program, examples, optimiser=optimiser, out_dir=tmp_path)
|
||||
return candidate, program, examples
|
||||
|
||||
|
||||
def test_instruction_scores_only_compile_changes_and_preserves_base(population, tmp_path):
|
||||
"""The optimiser and provenance see four compile changes, never the two held out."""
|
||||
|
||||
def checked(program, examples, metric):
|
||||
assert [e.change_id for e in examples] == ["c0-0", "c0-1", "c1-0", "c1-1"]
|
||||
for example in examples:
|
||||
assert "change" not in example.inputs()
|
||||
assert "PRIVATE" not in json.dumps(example.toDict())
|
||||
return instruction(program, examples, metric)
|
||||
|
||||
candidate, base, examples = run_round(population, tmp_path, checked)
|
||||
assert candidate.program_id != hp.program_id(base)
|
||||
assert base.hunt.signature.instructions == hp.BASELINE_INSTRUCTION
|
||||
assert candidate.compile_score["recall"] == 1
|
||||
assert candidate.compile_score["ci"] is not None
|
||||
assert candidate.compile_score["positive_units"] == 2
|
||||
assert candidate.compile_score["paired"]["recall"]["diff"] == 1
|
||||
assert candidate.compile_score["paired"]["recall"]["groups"] == 2
|
||||
provenance = json.loads((tmp_path / "proposer_inputs.json").read_text())
|
||||
assert provenance == [{"change_id": e.change_id, "fold": e.fold} for e in examples]
|
||||
assert "c2-" not in json.dumps(provenance)
|
||||
saved = hp.HuntProgram()
|
||||
saved.load_state(candidate.state)
|
||||
assert hp.program_id(saved, mask=candidate.mask) == candidate.program_id
|
||||
|
||||
|
||||
def test_two_surfaces_are_refused(population, tmp_path):
|
||||
"""Instruction and demo edits cannot enter the same round."""
|
||||
|
||||
def two(program, examples, metric):
|
||||
instruction(program, examples, metric)
|
||||
program.hunt.demos = [dspy.Example(**examples[0].inputs(), findings=[])]
|
||||
return program
|
||||
|
||||
with pytest.raises(ValueError, match="one"):
|
||||
run_round(population, tmp_path, two)
|
||||
assert not (tmp_path / "candidates.jsonl").exists()
|
||||
|
||||
|
||||
def test_history_contains_all_candidates_but_no_holdout(population, tmp_path):
|
||||
"""Acceptance records in the same directory cannot leak through the history reader."""
|
||||
candidate, _, _ = run_round(population, tmp_path)
|
||||
|
||||
def other_instruction(program, examples, metric):
|
||||
"""Produce a distinct second candidate for the same compile population."""
|
||||
program.hunt.signature = program.hunt.signature.with_instructions("Follow the changed value into the interpreter.")
|
||||
return program
|
||||
|
||||
second, _, _ = run_round(population, tmp_path, other_instruction)
|
||||
(tmp_path / "comparison.json").write_text('{"holdout": 987.654}')
|
||||
(tmp_path / "rejected.jsonl").write_text(json.dumps({"program_id": second.program_id, "holdout": 987.654}) + "\n")
|
||||
history = visible_to_proposer(tmp_path)
|
||||
assert [record["program_id"] for record in history] == [candidate.program_id, second.program_id]
|
||||
assert history[0]["compile_score"]["recall"] == 1
|
||||
assert "holdout" not in json.dumps(history) and "987.654" not in json.dumps(history)
|
||||
|
||||
|
||||
def test_demo_and_mask_rounds(population, tmp_path):
|
||||
"""Single demo and mask changes retain their actual provenance."""
|
||||
|
||||
def demos(program, examples, metric):
|
||||
program.hunt.demos = [dspy.Example(**examples[0].inputs(), findings=[])]
|
||||
return program
|
||||
|
||||
demo, _, _ = run_round(population, tmp_path / "demos", demos, "demos")
|
||||
assert demo.change == {"demos": {"count": 1, "change_ids": ["c0-0"]}}
|
||||
mask, _, _ = run_round(population, tmp_path / "mask", lambda p, e, m: hp.HuntProgram(mask=["roles"]), "mask")
|
||||
assert mask.change == {"mask": ["roles"]}
|
||||
|
||||
|
||||
def test_examples_are_not_an_editable_answer_key(population, tmp_path):
|
||||
"""An optimiser's mutation of an example cannot change the paired scoring population."""
|
||||
|
||||
def mutate(program, examples, metric):
|
||||
examples[0].change["kind"] = "ordinary"
|
||||
return instruction(program, examples, metric)
|
||||
|
||||
candidate, _, examples = run_round(population, tmp_path, mutate)
|
||||
assert candidate.compile_score["positive_units"] == 2
|
||||
assert examples[0].change["kind"] == "vulnerability"
|
||||
|
||||
|
||||
def test_invalid_round_and_cross_fold_repository_are_refused(population):
|
||||
"""A round has one kind, and a repository cannot straddle compile and holdout."""
|
||||
with pytest.raises(ValueError, match="kind"):
|
||||
Round(("instruction", "demos"), "base", [0], 0, "callable", {"usd": 1, "calls": 10})
|
||||
changes, rows, render = deepcopy(population)
|
||||
changes[-1]["repo"] = changes[0]["repo"]
|
||||
with pytest.raises(ValueError, match="repository"):
|
||||
compile_examples(changes, rows, compile_folds=[0, 1], render=render)
|
||||
|
||||
|
||||
def test_registry_resolves_dspy_classes():
|
||||
"""Registry lookups load the real DSPy optimiser classes."""
|
||||
assert OPTIMISERS["gepa"] is dspy.GEPA
|
||||
assert OPTIMISERS["miprov2"] is dspy.MIPROv2
|
||||
assert OPTIMISERS["bootstrap"] is dspy.BootstrapFewShot
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["gepa", "miprov2", "bootstrap", "configured-miprov2"])
|
||||
def test_named_optimisers_construct_and_receive_only_compile_examples(population, tmp_path, monkeypatch, name):
|
||||
"""Exercise real constructors and the compile call without running a model search."""
|
||||
changes, rows, render = population
|
||||
examples = compile_examples(changes, rows, compile_folds=[0, 1], render=render)
|
||||
base = hp.HuntProgram()
|
||||
called = []
|
||||
|
||||
def compile(self, student, *, trainset, **kwargs):
|
||||
"""Inspect the real optimiser and supply a deterministic compiled result."""
|
||||
called.append(type(self))
|
||||
assert {e.fold for e in trainset} == {0, 1}
|
||||
assert all("change" not in e.inputs() for e in trainset)
|
||||
if name == "gepa":
|
||||
feedback = self.metric_fn(trainset[0], dspy.Prediction(findings=[]))
|
||||
assert feedback.score == 0 and "missed" in feedback.feedback
|
||||
if name.endswith("miprov2"):
|
||||
assert kwargs == {"data_aware_proposer": False}
|
||||
return instruction(student, trainset, lambda *args: 0)
|
||||
|
||||
cls = OPTIMISERS[name.removeprefix("configured-")]
|
||||
monkeypatch.setattr(cls, "compile", compile)
|
||||
round = Round("instruction", hp.program_id(base), [0, 1], 13, name, {"usd": 1, "calls": 100})
|
||||
with dspy.context(lm=DummyLM([{"findings": "[]"}] * 8), adapter=dspy.ChatAdapter()):
|
||||
optimiser = cls(metric=lambda *args: 0) if name.startswith("configured-") else name
|
||||
candidate = propose(round, base, examples, optimiser=optimiser, out_dir=tmp_path)
|
||||
assert called == [cls] and candidate.compile_score["ci"]
|
||||
|
||||
|
||||
def test_gepa_without_model_fails_clearly(population, tmp_path):
|
||||
"""A missing reflection model is diagnosed before an optimiser is launched."""
|
||||
changes, rows, render = population
|
||||
examples = compile_examples(changes, rows, compile_folds=[0, 1], render=render)
|
||||
base = hp.HuntProgram()
|
||||
round = Round("instruction", hp.program_id(base), [0, 1], 13, "gepa", {"usd": 1, "calls": 100})
|
||||
with dspy.context(lm=None), pytest.raises(ValueError, match="reflection model"):
|
||||
propose(round, base, examples, optimiser="gepa", out_dir=tmp_path)
|
||||
Reference in New Issue
Block a user