diff --git a/benchmarks/unseen/extract.py b/benchmarks/unseen/extract.py index dc6d9f2..0bdc101 100644 --- a/benchmarks/unseen/extract.py +++ b/benchmarks/unseen/extract.py @@ -494,7 +494,20 @@ def ordinary_history(git, head: str) -> list[dict]: return rows -def draw_ordinary(rows: list[dict], fixes: set[str], known_functions: set[tuple[str, str]], *, seed: int) -> tuple[list[dict], dict]: +ORDINARY_FLOOR = 40 # eligible ordinary commits a repository must have before one stratified draw of 12 + + +def draw_ordinary( + rows: list[dict], + fixes: set[str], + known_functions: set[tuple[str, str]], + *, + seed: int, + floor: int = ORDINARY_FLOOR, + draws: int = 12, +) -> tuple[list[dict], dict]: + """One stratified draw of ``draws`` ordinary commits (time third x size bucket, proportional, at least one per + third). ``draws`` equal to the eligible count takes every eligible commit, cell by cell.""" import random from collections import Counter, defaultdict @@ -505,7 +518,9 @@ def draw_ordinary(rows: list[dict], fixes: set[str], known_functions: set[tuple[ counts[reason or "eligible"] += 1 if reason is None: eligible.append(row) - if len(eligible) < 40: + if floor < 12 or draws < 12: + raise ValueError("the ordinary floor and draw size cannot be below one draw of 12") + if len(eligible) < max(floor, draws): failure = Rejected("ordinary_floor") failure.counts = dict(counts) raise failure @@ -523,11 +538,11 @@ def draw_ordinary(rows: list[dict], fixes: set[str], known_functions: set[tuple[ # Largest-deficit rounding with the mandatory minimum of one per time third. # No cell can receive more draws than it has eligible commits. allocation = Counter() - targets = {cell: 12 * len(items) / len(eligible) for cell, items in cells.items()} + targets = {cell: draws * len(items) / len(eligible) for cell, items in cells.items()} for third in range(3): cell = max(sorted(c for c in cells if c[0] == third), key=lambda c: targets[c]) allocation[cell] = 1 - while sum(allocation.values()) < 12: + while sum(allocation.values()) < draws: cell = max(sorted(c for c in cells if allocation[c] < len(cells[c])), key=lambda c: targets[c] - allocation[c]) allocation[cell] += 1 rng = random.Random(seed) diff --git a/benchmarks/valid/assemble.py b/benchmarks/valid/assemble.py index 5c399c7..efb0a9f 100644 --- a/benchmarks/valid/assemble.py +++ b/benchmarks/valid/assemble.py @@ -18,20 +18,27 @@ from .catalog import local_git from .verify import verdict -def negatives(rows: list[dict], fixes: set[str], functions: set[tuple[str, str]], *, count: int, seed: int): - """Combine the shipped twelve-commit stratified draws without duplicates.""" +def negatives( + rows: list[dict], fixes: set[str], functions: set[tuple[str, str]], *, count: int, seed: int, floor: int = extract.ORDINARY_FLOOR +): + """Combine the shipped twelve-commit stratified draws without duplicates. + + Small cells can stay unallocated in every twelve-commit draw, so when the union cannot reach ``count`` (a + training repository whose eligible commits barely cover the ratio) one stratified draw of ``count`` is used + instead; with ``count`` equal to the eligible commits that is every eligible commit.""" available = {row["head"] for row in rows if extract.ordinary_rejection(row, fixes, functions) is None} if len(available) < count: raise ValueError(f"insufficient ordinary commits: {len(available)} available, {count} required") chosen = {} # Every call retains the original population and its time/size strata. for attempt in range(max(100, math.ceil(count / 12) * 100)): - drawn, exclusions = extract.draw_ordinary(rows, fixes, functions, seed=seed + attempt) + drawn, exclusions = extract.draw_ordinary(rows, fixes, functions, seed=seed + attempt, floor=floor) for row in drawn: chosen.setdefault(row["head"], row) if len(chosen) == count: return sorted(chosen.values(), key=lambda row: row["head"]), exclusions - raise ValueError("ordinary draw did not reach the required distinct count") + drawn, exclusions = extract.draw_ordinary(rows, fixes, functions, seed=seed, floor=floor, draws=count) + return sorted(drawn, key=lambda row: row["head"]), exclusions def freeze( @@ -45,9 +52,17 @@ def freeze( ratio: int = 20, history_heads: dict | None = None, fix_index: dict | None = None, + floor: int = extract.ORDINARY_FLOOR, ) -> dict: + """Freeze accepted positives with ``ratio`` ordinary pushes per fix commit from the same repositories. + + ``floor`` is the eligible-commit count a repository needs before its ordinary pushes are drawn. The + held-out benchmark keeps the shipped 40; a training freeze may lower it (never below the ratio) so small + repositories with 20 eligible commits still contribute, and the record carries the floor used.""" if type(ratio) is not int or ratio < 20: raise ValueError("ordinary ratio must be at least 20") + if type(floor) is not int or floor < ratio: + raise ValueError("ordinary floor must be at least the ratio") if len({row["id"] for row in cases}) != len(cases) or len({row["id"] for row in verifications}) != len(verifications): raise ValueError("duplicate case or verification id") used_names = {normalize(name) for name in (set().union(*used.sources.values()) if hasattr(used, "sources") else used)} @@ -81,7 +96,7 @@ def freeze( # accepted functions in one fix collapse to one vulnerability change below, so the # negative count must track distinct (commit, parent) pairs. change_count = len({(case["commit"], case["parent"]) for case in positives}) - ordinary, exclusions = negatives(history, fixes, functions, count=ratio * change_count, seed=seed) + ordinary, exclusions = negatives(history, fixes, functions, count=ratio * change_count, seed=seed, floor=floor) draws[repo] = exclusions changes = [] pairs = defaultdict(list) @@ -137,6 +152,7 @@ def freeze( history_heads=heads, ordinary_exclusions=draws, negative_ratio=ratio, + ordinary_floor=floor, label_check_counts={"confirmed": sum(len(v) for v in retained.values()), "unchecked": 0, "unclear": 0, "refactor": 0}, ) payloads = [ diff --git a/tests/test_unseen_extract.py b/tests/test_unseen_extract.py index f0ec2c3..9a1fbad 100644 --- a/tests/test_unseen_extract.py +++ b/tests/test_unseen_extract.py @@ -406,6 +406,9 @@ def test_draw_seed_proportional_cells_floor_and_thirds(): assert counts["eligible"] == 120 with pytest.raises(s.Rejected, match="ordinary_floor"): e.draw_ordinary(rows[:39], set(), set(), seed=19) + assert len(e.draw_ordinary(rows[:39], set(), set(), seed=19, floor=20)[0]) == 12 # a training freeze may lower the floor + with pytest.raises(ValueError, match="below one draw"): + e.draw_ordinary(rows, set(), set(), seed=19, floor=11) # Thirds come from the history's time span, not rank among eligible changes. with pytest.raises(s.Rejected, match="time_thirds"): e.draw_ordinary([ordinary(i) for i in range(40)] + [ordinary(10000, message="security")], set(), set(), seed=1) diff --git a/tests/test_valid_benchmark.py b/tests/test_valid_benchmark.py index 60bb557..91a546d 100644 --- a/tests/test_valid_benchmark.py +++ b/tests/test_valid_benchmark.py @@ -385,7 +385,18 @@ def test_insufficient_negatives_and_invalid_ratio(pair, tmp_path, monkeypatch): assemble.freeze(tmp_path, **kwargs) with pytest.raises(ValueError, match="at least 20"): assemble.freeze(tmp_path, ratio=19, **kwargs) + with pytest.raises(ValueError, match="at least the ratio"): + assemble.freeze(tmp_path, floor=19, **kwargs) assert not (tmp_path / "freeze-valid.json").exists() + # A training freeze lowers the floor to the ratio: 20 eligible commits now suffice, and the record says so. + monkeypatch.setattr(extract, "ordinary_history", lambda *args: ordinary_rows(20)) + with pytest.raises(ValueError, match="ordinary_floor"): # Rejected is a ValueError with a fixed reason code + assemble.freeze(tmp_path / "default", **kwargs) + result = assemble.freeze(tmp_path / "train", floor=20, **kwargs) + assert result["ordinary_floor"] == 20 and result["negative_ratio"] == 20 + # Twenty eligible commits for a ratio of twenty: every one of them is drawn, none twice. + ordinary = [c for c in replay.load_slice(tmp_path / "train/pool-valid.toml", 1) if c["kind"] == "ordinary"] + assert len(ordinary) == 20 and len({c["head"] for c in ordinary}) == 20 def test_freeze_cannot_overwrite_or_trust_verdict_alone(pair, tmp_path, monkeypatch):