feat(learning-harness): the acceptance rule and the directory revert

Task 3.6. A round is kept only when the family it targets does not go backwards
on either the train or the held-out pairs, at least one of them moves forward,
and no other family flips more pairs than that family showed against itself in
round zero. A tie is a rejection: accepting one would let the loop wander on
noise and call it progress. The rule lives in one place so no caller can soften
it.

A family with no measured floor gets no budget, because unmeasured is not the
same as unlimited and the alternative is a family that never ran in round zero
quietly absorbing every regression. A family marked as never gating cannot
reject a round however much it moves.

Undoing is a directory restored byte for byte: files the round added are
removed, files it deleted are put back, and a digest per family shows that every
directory other than the target is identical before and after, whatever the
outcome. That is what makes the freeze structural rather than promised.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/learning/acceptance.py

@@ -0,0 +1,117 @@
+"""When a round is kept, and how it is undone (learning-harness, Req 6.2, 9.5).
+
+A round is accepted only when the family it targets does not go backwards on either the train or the
+held-out pairs, at least one of them moves forward, and no other family flips more pairs than that
+family showed against itself in round zero. A tie is a rejection: accepting one would let the loop
+wander on noise and call it progress.
+
+A family with no measured floor gets no budget. Unmeasured is not the same as unlimited, and the
+alternative is a family that never ran in round zero quietly absorbing every regression.
+
+Undoing is a directory restored byte for byte, files the round added removed, files it deleted put back.
+A revert that only rewrites what it can see is not a revert.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+from .rounds import NoiseFloor
+
+
+@dataclass(frozen=True)
+class Verdict:
+    accepted: bool
+    reason: str
+    offenders: tuple[str, ...] = ()
+
+    def to_dict(self) -> dict[str, object]:
+        return {"accepted": self.accepted, "reason": self.reason, "offenders": list(self.offenders)}
+
+
+def decide(
+    *,
+    target: str,
+    train_delta: float,
+    holdout_delta: float,
+    sweep: Mapping[str, int],
+    floors: Mapping[str, NoiseFloor],
+) -> Verdict:
+    """The acceptance rule, in one place, so no caller can soften it."""
+    if train_delta < 0:
+        return Verdict(accepted=False, reason="train_regressed")
+    if holdout_delta < 0:
+        return Verdict(accepted=False, reason="holdout_regressed")
+    if train_delta == 0 and holdout_delta == 0:
+        return Verdict(accepted=False, reason="tie")
+    offenders = tuple(
+        family for family, flips in sorted(sweep.items()) if family != target and _gates(family, floors) and flips > _budget(family, floors)
+    )
+    if offenders:
+        return Verdict(accepted=False, reason="collateral_regression", offenders=offenders)
+    return Verdict(accepted=True, reason="improved")
+
+
+def _gates(family: str, floors: Mapping[str, NoiseFloor]) -> bool:
+    floor = floors.get(family)
+    return floor.gates if floor is not None else True
+
+
+def _budget(family: str, floors: Mapping[str, NoiseFloor]) -> int:
+    floor = floors.get(family)
+    return floor.budget_flips if floor is not None else 0
+
+
+def within_budget(spent_usd: float, cap_usd: float) -> bool:
+    """A round may spend up to its cap; crossing it reverts the round rather than finishing it."""
+    return spent_usd <= cap_usd
+
+
+@dataclass(frozen=True)
+class DirectorySnapshot:
+    """Every file of one directory as it was, so a rejected round leaves nothing behind."""
+
+    directory: Path
+    files: tuple[tuple[str, bytes], ...]
+
+    @staticmethod
+    def of(directory: Path) -> DirectorySnapshot:
+        entries = [(path.relative_to(directory).as_posix(), path.read_bytes()) for path in sorted(directory.rglob("*")) if path.is_file()]
+        return DirectorySnapshot(directory=directory, files=tuple(entries))
+
+    def restore(self) -> None:
+        kept = {name for name, _ in self.files}
+        for path in sorted(self.directory.rglob("*"), reverse=True):
+            if path.is_file() and path.relative_to(self.directory).as_posix() not in kept:
+                path.unlink()
+        for name, content in self.files:
+            target = self.directory / name
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(content)
+
+
+def digest_configs(configs_dir: Path) -> dict[str, str]:
+    """One digest per family directory, so an untouched family can be shown to be untouched."""
+    digests: dict[str, str] = {}
+    for directory in sorted(path for path in configs_dir.iterdir() if path.is_dir()) if configs_dir.is_dir() else []:
+        digest = hashlib.sha256()
+        for path in sorted(directory.rglob("*")):
+            if path.is_file():
+                digest.update(path.relative_to(directory).as_posix().encode())
+                digest.update(path.read_bytes())
+        digests[directory.name] = digest.hexdigest()
+    return digests
+
+
+def bump_version(directory: Path, version: str) -> None:
+    """Write the accepted version into this family's settings, and only this family's."""
+    settings = directory / "family.toml"
+    lines = settings.read_text(encoding="utf-8").splitlines()
+    rewritten = [f'version = "{version}"' if line.strip().startswith("version") else line for line in lines]
+    settings.write_text("\n".join(rewritten) + "\n", encoding="utf-8")
+
+
+__all__ = ["DirectorySnapshot", "Verdict", "bump_version", "decide", "digest_configs", "within_budget"]

tests/test_learning_acceptance.py

@@ -0,0 +1,115 @@
+"""learning-harness task 3.6: the acceptance rule and the directory revert (offline)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from openultrasast.learning.rounds import NoiseFloor
+
+
+def _floor(family: str, budget: int = 0, *, gates: bool = True) -> NoiseFloor:
+    return NoiseFloor(family=family, k_runs=5, pairs=10, flaky_pairs=budget, negative_flip_rate=0.0, budget_flips=budget, gates=gates)
+
+
+def _decide(train: float, holdout: float, sweep: dict[str, int], floors: dict[str, NoiseFloor]):  # type: ignore[no-untyped-def]
+    from openultrasast.learning.acceptance import decide
+
+    return decide(target="access_control", train_delta=train, holdout_delta=holdout, sweep=sweep, floors=floors)
+
+
+def test_a_change_that_helps_and_hurts_nobody_is_accepted() -> None:
+    verdict = _decide(0.1, 0.2, {"injection": 0}, {"injection": _floor("injection")})
+    assert verdict.accepted and verdict.reason == "improved"
+
+
+def test_one_side_may_stand_still_but_the_other_must_move() -> None:
+    assert _decide(0.0, 0.2, {}, {}).accepted
+    assert _decide(0.2, 0.0, {}, {}).accepted
+    tie = _decide(0.0, 0.0, {}, {})
+    assert not tie.accepted and tie.reason == "tie"  # a tie is rejected, or the loop drifts on noise
+
+
+def test_a_decrease_on_either_side_is_rejected() -> None:
+    train_down = _decide(-0.1, 0.5, {}, {})
+    assert not train_down.accepted and train_down.reason == "train_regressed"
+    holdout_down = _decide(0.5, -0.1, {}, {})
+    assert not holdout_down.accepted and holdout_down.reason == "holdout_regressed"
+
+
+def test_another_family_beyond_its_measured_budget_rejects_the_round() -> None:
+    within = _decide(0.2, 0.2, {"injection": 2}, {"injection": _floor("injection", 2)})
+    assert within.accepted  # inside the noise this family shows against itself
+    beyond = _decide(0.2, 0.2, {"injection": 3}, {"injection": _floor("injection", 2)})
+    assert not beyond.accepted and beyond.reason == "collateral_regression"
+    assert beyond.offenders == ("injection",)
+
+
+def test_a_family_that_never_gates_cannot_reject_a_round() -> None:
+    verdict = _decide(0.2, 0.2, {"memory": 99}, {"memory": _floor("memory", 0, gates=False)})
+    assert verdict.accepted and verdict.offenders == ()
+
+
+def test_a_family_with_no_measured_floor_gets_no_budget() -> None:
+    verdict = _decide(0.2, 0.2, {"path": 1}, {})
+    assert not verdict.accepted and verdict.offenders == ("path",)  # unmeasured is not the same as unlimited
+
+
+def test_a_snapshot_restores_a_family_directory_byte_for_byte(tmp_path: Path) -> None:
+    from openultrasast.learning.acceptance import DirectorySnapshot
+
+    directory = tmp_path / "access_control"
+    directory.mkdir()
+    (directory / "prompt.md").write_text("original\n")
+    (directory / "family.toml").write_text('version = "0"\n')
+    snapshot = DirectorySnapshot.of(directory)
+    (directory / "prompt.md").write_text("changed\n")
+    (directory / "extra.md").write_text("added by a round\n")
+    (directory / "family.toml").unlink()
+    snapshot.restore()
+    assert (directory / "prompt.md").read_text() == "original\n"
+    assert (directory / "family.toml").read_text() == 'version = "0"\n'
+    assert not (directory / "extra.md").exists()  # a file the round added is not left behind
+    assert sorted(path.name for path in directory.iterdir()) == ["family.toml", "prompt.md"]
+
+
+def test_every_other_family_directory_is_untouched_across_a_round(tmp_path: Path) -> None:
+    """Req 6.2: a round can only write its target, so the freeze is structural rather than promised."""
+    from openultrasast.learning.acceptance import DirectorySnapshot, digest_configs
+    from openultrasast.learning.detectors import write_default_configs
+    from openultrasast.learning.families import load_families
+    from openultrasast.learning.proposer import Proposal, apply_proposal
+
+    configs = tmp_path / "configs"
+    write_default_configs(configs, load_families(), prompt="p", version="0")
+    before = digest_configs(configs)
+    snapshot = DirectorySnapshot.of(configs / "access_control")
+    apply_proposal(Proposal(hypothesis="h", lever="checklist", change={"checklist.md": "- new\n"}), configs / "access_control")
+    after = digest_configs(configs)
+    assert after["access_control"] != before["access_control"]
+    assert {name: value for name, value in after.items() if name != "access_control"} == {
+        name: value for name, value in before.items() if name != "access_control"
+    }
+    snapshot.restore()
+    assert digest_configs(configs) == before  # a rejected round leaves the whole tree as it found it
+
+
+def test_a_bumped_version_is_written_only_on_acceptance(tmp_path: Path) -> None:
+    from openultrasast.learning.acceptance import bump_version
+    from openultrasast.learning.detectors import load_family_configs, write_default_configs
+    from openultrasast.learning.families import load_families
+
+    configs = tmp_path / "configs"
+    write_default_configs(configs, load_families(), prompt="p", version="0")
+    bump_version(configs / "access_control", "1")
+    taxonomy = load_families()
+    assert load_family_configs(configs, taxonomy)["access_control"].version == "1"
+    assert load_family_configs(configs, taxonomy)["injection"].version == "0"
+
+
+@pytest.mark.parametrize(("spent", "cap", "expected"), [(0.5, 10.0, True), (10.0, 10.0, True), (10.01, 10.0, False)])
+def test_a_round_over_its_cost_cap_is_reverted(spent: float, cap: float, expected: bool) -> None:
+    from openultrasast.learning.acceptance import within_budget
+
+    assert within_budget(spent, cap) is expected