bench(valid): a training freeze may cap the history walk at the newest N first-parent commits

ordinary_history(limit=) walks only the newest commits; assemble.freeze(history_limit=) and
--history-limit pass it through and the record carries it. The held-out benchmark passes none.
Reason: trigger.dev's 7,109-commit walk runs for hours at the per-commit cost of the function
and generated checks; the training freeze now walks its newest 1,500.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/unseen/extract.py

@@ -486,9 +486,15 @@ def _history_cache(git, cache_dir: Path | None):
         yield cached, handle
 
 
-def ordinary_history(git, head: str, *, cache_dir: Path | None = None) -> list[dict]:
+def ordinary_history(git, head: str, *, cache_dir: Path | None = None, limit: int | None = None) -> list[dict]:
-    """Read first-parent history, optionally reusing completed rows from this clone."""
+    """Read first-parent history, optionally reusing completed rows from this clone.
-    raw = git.run("log", "--first-parent", "--format=%H%x00%P%x00%ct%x00%an%x00%B%x1e", head)
+
+    ``limit`` walks only the newest ``limit`` first-parent commits (a training freeze's cap; the held-out
+    benchmark passes none)."""
+    if limit is not None and (type(limit) is not int or limit < 1):
+        raise ValueError("history limit must be a positive integer")
+    count = () if limit is None else ("-n", str(limit))
+    raw = git.run("log", "--first-parent", *count, "--format=%H%x00%P%x00%ct%x00%an%x00%B%x1e", head)
     rows = []
     with _history_cache(git, cache_dir) as (cached, handle):
         for entry in raw.split("\x1e"):

benchmarks/valid/assemble.py

@@ -54,13 +54,16 @@ def freeze(
     history_cache: Path | None = None,
     fix_index: dict | None = None,
     floor: int = extract.ORDINARY_FLOOR,
+    history_limit: int | None = None,
 ) -> dict:
     """Freeze accepted positives with ``ratio`` ordinary pushes per fix commit from the same repositories.
 
     ``floor`` is the eligible-commit count a repository needs before its ordinary pushes are drawn. The
     held-out benchmark keeps the shipped 40; a training freeze may lower it (never below the ratio) so small
     repositories with 20 eligible commits still contribute, and the record carries the floor used.
-    ``history_cache`` reuses completed per-commit history rows across freezes of the same clone."""
+    ``history_cache`` reuses completed per-commit history rows across freezes of the same clone. ``history_limit``
+    caps the walk at the newest first-parent commits for a training freeze and is recorded; the held-out benchmark
+    passes none."""
     if type(ratio) is not int or ratio < 20:
         raise ValueError("ordinary ratio must be at least 20")
     if type(floor) is not int or floor < ratio:
@@ -94,6 +97,8 @@ def freeze(
         head = (history_heads or {}).get(repo) or git.run("rev-parse", "HEAD").strip()
         heads[repo] = head
         history_options = {"cache_dir": history_cache} if history_cache is not None else {}
+        if history_limit is not None:
+            history_options["limit"] = history_limit
         history = sorted(extract.ordinary_history(git, head, **history_options), key=lambda row: row["head"])
         # Ratio is per positive CHANGE (a fix commit), not per accepted function. Several
         # accepted functions in one fix collapse to one vulnerability change below, so the
@@ -156,6 +161,7 @@ def freeze(
         ordinary_exclusions=draws,
         negative_ratio=ratio,
         ordinary_floor=floor,
+        history_limit=history_limit,
         label_check_counts={"confirmed": sum(len(v) for v in retained.values()), "unchecked": 0, "unclear": 0, "refactor": 0},
     )
     payloads = [
@@ -181,6 +187,7 @@ def main(argv=None):
     parser.add_argument("--out", type=Path, required=True)
     parser.add_argument("--seed", type=int, default=0)
     parser.add_argument("--history-cache", type=Path, help="Directory for per-clone commit history caches")
+    parser.add_argument("--history-limit", type=int, default=None, help="Training only: walk the newest N first-parent commits")
     args = parser.parse_args(argv)
     result = freeze(
         args.out,
@@ -190,6 +197,7 @@ def main(argv=None):
         used=json.loads(args.used.read_text()),
         seed=args.seed,
         history_cache=args.history_cache,
+        history_limit=args.history_limit,
     )
     print(json.dumps(result, sort_keys=True))
     return 0

tests/test_unseen_extract.py

@@ -670,3 +670,12 @@ def test_partial_clone_lazy_fetch_preserves_extraction_content(history, tmp_path
     assert partial.files_read > 0 and partial.bytes_read > 0
     assert json.dumps(result, sort_keys=True).encode() == json.dumps(baseline, sort_keys=True).encode()
     assert sum("built-in: git fetch " in line for line in trace.read_text().splitlines()) >= 12
+
+
+def test_history_limit_walks_only_the_newest_commits(history):
+    git, *_ = history
+    full = e.ordinary_history(git, "HEAD")
+    assert len(full) > 2
+    assert e.ordinary_history(git, "HEAD", limit=2) == full[:2]  # the newest first-parent commits, same rows
+    with pytest.raises(ValueError, match="positive integer"):
+        e.ordinary_history(git, "HEAD", limit=0)

tests/test_valid_benchmark.py

@@ -448,7 +448,7 @@ def test_insufficient_negatives_and_invalid_ratio(pair, tmp_path, monkeypatch):
     with pytest.raises(ValueError, match="ordinary_floor"):  # Rejected is a ValueError with a fixed reason code
         assemble.freeze(tmp_path / "default", **kwargs)
     result = assemble.freeze(tmp_path / "train", floor=20, **kwargs)
-    assert result["ordinary_floor"] == 20 and result["negative_ratio"] == 20
+    assert result["ordinary_floor"] == 20 and result["negative_ratio"] == 20 and result["history_limit"] is None
     # Twenty eligible commits for a ratio of twenty: every one of them is drawn, none twice.
     ordinary = [c for c in replay.load_slice(tmp_path / "train/pool-valid.toml", 1) if c["kind"] == "ordinary"]
     assert len(ordinary) == 20 and len({c["head"] for c in ordinary}) == 20