bench(valid): a training freeze may lower the ordinary floor to the ratio; the draw takes a size

The held-out benchmark keeps the 40-commit floor and the union of twelve-commit stratified draws.
A training freeze passes floor=20 (never below the ratio) and the record carries ordinary_floor.
When the union of twelve-commit draws cannot reach the count (small cells stay unallocated on
every draw), one stratified draw of the full count is used; with count equal to the eligible
commits that is every one of them.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-13 18:33:10 +02:00
co-authored by Claude Fable 5.1
parent f29b3a3df2
commit bbbb0f397e
4 changed files with 54 additions and 9 deletions
+3
View File
@@ -406,6 +406,9 @@ def test_draw_seed_proportional_cells_floor_and_thirds():
assert counts["eligible"] == 120
with pytest.raises(s.Rejected, match="ordinary_floor"):
e.draw_ordinary(rows[:39], set(), set(), seed=19)
assert len(e.draw_ordinary(rows[:39], set(), set(), seed=19, floor=20)[0]) == 12 # a training freeze may lower the floor
with pytest.raises(ValueError, match="below one draw"):
e.draw_ordinary(rows, set(), set(), seed=19, floor=11)
# Thirds come from the history's time span, not rank among eligible changes.
with pytest.raises(s.Rejected, match="time_thirds"):
e.draw_ordinary([ordinary(i) for i in range(40)] + [ordinary(10000, message="security")], set(), set(), seed=1)
+11
View File
@@ -385,7 +385,18 @@ def test_insufficient_negatives_and_invalid_ratio(pair, tmp_path, monkeypatch):
assemble.freeze(tmp_path, **kwargs)
with pytest.raises(ValueError, match="at least 20"):
assemble.freeze(tmp_path, ratio=19, **kwargs)
with pytest.raises(ValueError, match="at least the ratio"):
assemble.freeze(tmp_path, floor=19, **kwargs)
assert not (tmp_path / "freeze-valid.json").exists()
# A training freeze lowers the floor to the ratio: 20 eligible commits now suffice, and the record says so.
monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows(20))
with pytest.raises(ValueError, match="ordinary_floor"): # Rejected is a ValueError with a fixed reason code
assemble.freeze(tmp_path / "default", **kwargs)
result = assemble.freeze(tmp_path / "train", floor=20, **kwargs)
assert result["ordinary_floor"] == 20 and result["negative_ratio"] == 20
# Twenty eligible commits for a ratio of twenty: every one of them is drawn, none twice.
ordinary = [c for c in replay.load_slice(tmp_path / "train/pool-valid.toml", 1) if c["kind"] == "ordinary"]
assert len(ordinary) == 20 and len({c["head"] for c in ordinary}) == 20
def test_freeze_cannot_overwrite_or_trust_verdict_alone(pair, tmp_path, monkeypatch):