bench(valid): a training freeze may lower the ordinary floor to the ratio; the draw takes a size

The held-out benchmark keeps the 40-commit floor and the union of twelve-commit stratified draws.
A training freeze passes floor=20 (never below the ratio) and the record carries ordinary_floor.
When the union of twelve-commit draws cannot reach the count (small cells stay unallocated on
every draw), one stratified draw of the full count is used; with count equal to the eligible
commits that is every one of them.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/unseen/extract.py

@@ -494,7 +494,20 @@ def ordinary_history(git, head: str) -> list[dict]:
     return rows
 
 
-def draw_ordinary(rows: list[dict], fixes: set[str], known_functions: set[tuple[str, str]], *, seed: int) -> tuple[list[dict], dict]:
+ORDINARY_FLOOR = 40  # eligible ordinary commits a repository must have before one stratified draw of 12
+
+
+def draw_ordinary(
+    rows: list[dict],
+    fixes: set[str],
+    known_functions: set[tuple[str, str]],
+    *,
+    seed: int,
+    floor: int = ORDINARY_FLOOR,
+    draws: int = 12,
+) -> tuple[list[dict], dict]:
+    """One stratified draw of ``draws`` ordinary commits (time third x size bucket, proportional, at least one per
+    third). ``draws`` equal to the eligible count takes every eligible commit, cell by cell."""
     import random
     from collections import Counter, defaultdict
 
@@ -505,7 +518,9 @@ def draw_ordinary(rows: list[dict], fixes: set[str], known_functions: set[tuple[
         counts[reason or "eligible"] += 1
         if reason is None:
             eligible.append(row)
-    if len(eligible) < 40:
+    if floor < 12 or draws < 12:
+        raise ValueError("the ordinary floor and draw size cannot be below one draw of 12")
+    if len(eligible) < max(floor, draws):
         failure = Rejected("ordinary_floor")
         failure.counts = dict(counts)
         raise failure
@@ -523,11 +538,11 @@ def draw_ordinary(rows: list[dict], fixes: set[str], known_functions: set[tuple[
     # Largest-deficit rounding with the mandatory minimum of one per time third.
     # No cell can receive more draws than it has eligible commits.
     allocation = Counter()
-    targets = {cell: 12 * len(items) / len(eligible) for cell, items in cells.items()}
+    targets = {cell: draws * len(items) / len(eligible) for cell, items in cells.items()}
     for third in range(3):
         cell = max(sorted(c for c in cells if c[0] == third), key=lambda c: targets[c])
         allocation[cell] = 1
-    while sum(allocation.values()) < 12:
+    while sum(allocation.values()) < draws:
         cell = max(sorted(c for c in cells if allocation[c] < len(cells[c])), key=lambda c: targets[c] - allocation[c])
         allocation[cell] += 1
     rng = random.Random(seed)

benchmarks/valid/assemble.py

@@ -18,20 +18,27 @@ from .catalog import local_git
 from .verify import verdict
 
 
-def negatives(rows: list[dict], fixes: set[str], functions: set[tuple[str, str]], *, count: int, seed: int):
-    """Combine the shipped twelve-commit stratified draws without duplicates."""
+def negatives(
+    rows: list[dict], fixes: set[str], functions: set[tuple[str, str]], *, count: int, seed: int, floor: int = extract.ORDINARY_FLOOR
+):
+    """Combine the shipped twelve-commit stratified draws without duplicates.
+
+    Small cells can stay unallocated in every twelve-commit draw, so when the union cannot reach ``count`` (a
+    training repository whose eligible commits barely cover the ratio) one stratified draw of ``count`` is used
+    instead; with ``count`` equal to the eligible commits that is every eligible commit."""
     available = {row["head"] for row in rows if extract.ordinary_rejection(row, fixes, functions) is None}
     if len(available) < count:
         raise ValueError(f"insufficient ordinary commits: {len(available)} available, {count} required")
     chosen = {}
     # Every call retains the original population and its time/size strata.
     for attempt in range(max(100, math.ceil(count / 12) * 100)):
-        drawn, exclusions = extract.draw_ordinary(rows, fixes, functions, seed=seed + attempt)
+        drawn, exclusions = extract.draw_ordinary(rows, fixes, functions, seed=seed + attempt, floor=floor)
         for row in drawn:
             chosen.setdefault(row["head"], row)
             if len(chosen) == count:
                 return sorted(chosen.values(), key=lambda row: row["head"]), exclusions
-    raise ValueError("ordinary draw did not reach the required distinct count")
+    drawn, exclusions = extract.draw_ordinary(rows, fixes, functions, seed=seed, floor=floor, draws=count)
+    return sorted(drawn, key=lambda row: row["head"]), exclusions
 
 
 def freeze(
@@ -45,9 +52,17 @@ def freeze(
     ratio: int = 20,
     history_heads: dict | None = None,
     fix_index: dict | None = None,
+    floor: int = extract.ORDINARY_FLOOR,
 ) -> dict:
+    """Freeze accepted positives with ``ratio`` ordinary pushes per fix commit from the same repositories.
+
+    ``floor`` is the eligible-commit count a repository needs before its ordinary pushes are drawn. The
+    held-out benchmark keeps the shipped 40; a training freeze may lower it (never below the ratio) so small
+    repositories with 20 eligible commits still contribute, and the record carries the floor used."""
     if type(ratio) is not int or ratio < 20:
         raise ValueError("ordinary ratio must be at least 20")
+    if type(floor) is not int or floor < ratio:
+        raise ValueError("ordinary floor must be at least the ratio")
     if len({row["id"] for row in cases}) != len(cases) or len({row["id"] for row in verifications}) != len(verifications):
         raise ValueError("duplicate case or verification id")
     used_names = {normalize(name) for name in (set().union(*used.sources.values()) if hasattr(used, "sources") else used)}
@@ -81,7 +96,7 @@ def freeze(
         # accepted functions in one fix collapse to one vulnerability change below, so the
         # negative count must track distinct (commit, parent) pairs.
         change_count = len({(case["commit"], case["parent"]) for case in positives})
-        ordinary, exclusions = negatives(history, fixes, functions, count=ratio * change_count, seed=seed)
+        ordinary, exclusions = negatives(history, fixes, functions, count=ratio * change_count, seed=seed, floor=floor)
         draws[repo] = exclusions
         changes = []
         pairs = defaultdict(list)
@@ -137,6 +152,7 @@ def freeze(
         history_heads=heads,
         ordinary_exclusions=draws,
         negative_ratio=ratio,
+        ordinary_floor=floor,
         label_check_counts={"confirmed": sum(len(v) for v in retained.values()), "unchecked": 0, "unclear": 0, "refactor": 0},
     )
     payloads = [

tests/test_unseen_extract.py

@@ -406,6 +406,9 @@ def test_draw_seed_proportional_cells_floor_and_thirds():
     assert counts["eligible"] == 120
     with pytest.raises(s.Rejected, match="ordinary_floor"):
         e.draw_ordinary(rows[:39], set(), set(), seed=19)
+    assert len(e.draw_ordinary(rows[:39], set(), set(), seed=19, floor=20)[0]) == 12  # a training freeze may lower the floor
+    with pytest.raises(ValueError, match="below one draw"):
+        e.draw_ordinary(rows, set(), set(), seed=19, floor=11)
     # Thirds come from the history's time span, not rank among eligible changes.
     with pytest.raises(s.Rejected, match="time_thirds"):
         e.draw_ordinary([ordinary(i) for i in range(40)] + [ordinary(10000, message="security")], set(), set(), seed=1)

tests/test_valid_benchmark.py

@@ -385,7 +385,18 @@ def test_insufficient_negatives_and_invalid_ratio(pair, tmp_path, monkeypatch):
         assemble.freeze(tmp_path, **kwargs)
     with pytest.raises(ValueError, match="at least 20"):
         assemble.freeze(tmp_path, ratio=19, **kwargs)
+    with pytest.raises(ValueError, match="at least the ratio"):
+        assemble.freeze(tmp_path, floor=19, **kwargs)
     assert not (tmp_path / "freeze-valid.json").exists()
+    # A training freeze lowers the floor to the ratio: 20 eligible commits now suffice, and the record says so.
+    monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows(20))
+    with pytest.raises(ValueError, match="ordinary_floor"):  # Rejected is a ValueError with a fixed reason code
+        assemble.freeze(tmp_path / "default", **kwargs)
+    result = assemble.freeze(tmp_path / "train", floor=20, **kwargs)
+    assert result["ordinary_floor"] == 20 and result["negative_ratio"] == 20
+    # Twenty eligible commits for a ratio of twenty: every one of them is drawn, none twice.
+    ordinary = [c for c in replay.load_slice(tmp_path / "train/pool-valid.toml", 1) if c["kind"] == "ordinary"]
+    assert len(ordinary) == 20 and len({c["head"] for c in ordinary}) == 20
 
 
 def test_freeze_cannot_overwrite_or_trust_verdict_alone(pair, tmp_path, monkeypatch):