bench(unseen): per-commit history cache for ordinary_history; assemble.freeze --history-cache

One append-only JSONL per clone (named by the sha256 of its top-level path), one line per commit
with the row as the walk built it and HISTORY_CACHE_VERSION, flushed after every commit so a
killed run keeps its progress. A cached row is reused only when its parents match the current log;
other versions and torn lines are recomputed. With no cache directory nothing changes. The
7,400-commit trigger.dev walk that took over an hour per freeze becomes a one-time cost.
Implemented by Codex (gpt-6-astra), gated here.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/unseen/extract.py

@@ -8,6 +8,7 @@ are dropped, and the recorded SZZ bias remains subject to task 2.6's label check
 from __future__ import annotations
 
 import ast
+import hashlib
 import json
 import re
 import signal
@@ -23,6 +24,8 @@ from .source import Candidate, Rejected
 EXTRACTION_CONTRACT = "v1"
 # Bump only when enumeration, eligibility or source.candidate outputs change.
 RESOLUTION_CONTRACT = "v1"
+# Bump whenever functions(), generated(), LANGUAGES or the row shape change: cached rows embed their results.
+HISTORY_CACHE_VERSION = 1
 
 LANGUAGES = {".py", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".php", ".java"}
 HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@", re.M)
@@ -452,45 +455,84 @@ def generated(git, revision: str, path: str, text: str) -> bool:
     return result
 
 
-def ordinary_history(git, head: str) -> list[dict]:
+@contextmanager
+def _history_cache(git, cache_dir: Path | None):
+    """Load completed rows and open the clone's cache for appending."""
+    if cache_dir is None:
+        yield {}, None
+        return
+    top = git.run("rev-parse", "--show-toplevel").strip()
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    path = cache_dir / (hashlib.sha256(top.encode()).hexdigest() + ".jsonl")
+    with path.open("a+b") as handle:
+        handle.seek(0)
+        cached = {}
+        line = b""
+        for line in handle:
+            try:
+                entry = json.loads(line)
+            except (json.JSONDecodeError, UnicodeDecodeError):
+                continue
+            if not isinstance(entry, dict) or entry.get("version") != HISTORY_CACHE_VERSION:
+                continue
+            row = entry.get("row")
+            sha = entry.get("head")
+            if isinstance(sha, str) and isinstance(row, dict) and row.get("head") == sha:
+                cached[sha] = row
+        if line and not line.endswith(b"\n"):
+            # Keep a partial last line from swallowing the next completed row.
+            handle.write(b"\n")
+            handle.flush()
+        yield cached, handle
+
+
+def ordinary_history(git, head: str, *, cache_dir: Path | None = None) -> list[dict]:
+    """Read first-parent history, optionally reusing completed rows from this clone."""
     raw = git.run("log", "--first-parent", "--format=%H%x00%P%x00%ct%x00%an%x00%B%x1e", head)
     rows = []
-    for entry in raw.split("\x1e"):
-        if not entry.strip():
-            continue
-        fields = entry.strip().split("\x00")
-        if len(fields) != 5 or not SHA.fullmatch(fields[0]):
-            raise InstrumentFailure("invalid_history")
-        sha, parents, timestamp, author, message = fields
-        row = {"head": sha, "parents": parents.split(), "timestamp": int(timestamp), "author": author, "message": message, "files": []}
-        rows.append(row)
-        if len(row["parents"]) != 1:
-            continue
-        base = row["parents"][0]
-        for path, added, removed in git.stats(base, sha):
-            hunks = git.hunks(base, sha, path)
-            info = {
-                "path": path,
-                "added": added,
-                "removed": removed,
-                "head_lines": [[max(1, n), max(1, n + max(c, 1) - 1)] for _, _, n, c in hunks],
-                "functions": [],
-                "generated": False,
-            }
-            if Path(path).suffix in LANGUAGES:
-                # Inspect both sides: deleting a vulnerable function is still touching it.
-                for revision, side in ((base, 0), (sha, 2)):
-                    text = git.blob(revision, path)
-                    if text is None:
-                        continue
-                    spans = functions(path, text)
-                    for hunk in hunks:
-                        start, count = hunk[side : side + 2]
-                        for name, lo, hi in spans:
-                            if max(1, start) <= hi and start + max(count, 1) - 1 >= lo:
-                                info["functions"].append(name)
-                    info["generated"] |= generated(git, revision, path, text)
-            row["files"].append(info)
+    with _history_cache(git, cache_dir) as (cached, handle):
+        for entry in raw.split("\x1e"):
+            if not entry.strip():
+                continue
+            fields = entry.strip().split("\x00")
+            if len(fields) != 5 or not SHA.fullmatch(fields[0]):
+                raise InstrumentFailure("invalid_history")
+            sha, parents, timestamp, author, message = fields
+            previous = cached.get(sha)
+            if previous is not None and previous.get("parents") == parents.split():
+                rows.append(previous)
+                continue
+            row = {"head": sha, "parents": parents.split(), "timestamp": int(timestamp), "author": author, "message": message, "files": []}
+            rows.append(row)
+            if len(row["parents"]) == 1:
+                base = row["parents"][0]
+                for path, added, removed in git.stats(base, sha):
+                    hunks = git.hunks(base, sha, path)
+                    info = {
+                        "path": path,
+                        "added": added,
+                        "removed": removed,
+                        "head_lines": [[max(1, n), max(1, n + max(c, 1) - 1)] for _, _, n, c in hunks],
+                        "functions": [],
+                        "generated": False,
+                    }
+                    if Path(path).suffix in LANGUAGES:
+                        # Inspect both sides: deleting a vulnerable function is still touching it.
+                        for revision, side in ((base, 0), (sha, 2)):
+                            text = git.blob(revision, path)
+                            if text is None:
+                                continue
+                            spans = functions(path, text)
+                            for hunk in hunks:
+                                start, count = hunk[side : side + 2]
+                                for name, lo, hi in spans:
+                                    if max(1, start) <= hi and start + max(count, 1) - 1 >= lo:
+                                        info["functions"].append(name)
+                            info["generated"] |= generated(git, revision, path, text)
+                    row["files"].append(info)
+            if handle is not None:
+                handle.write((json.dumps({"version": HISTORY_CACHE_VERSION, "head": sha, "row": row}) + "\n").encode())
+                handle.flush()
     return rows
 
 

benchmarks/valid/assemble.py

@@ -51,6 +51,7 @@ def freeze(
     seed: int = 0,
     ratio: int = 20,
     history_heads: dict | None = None,
+    history_cache: Path | None = None,
     fix_index: dict | None = None,
     floor: int = extract.ORDINARY_FLOOR,
 ) -> dict:
@@ -58,7 +59,8 @@ def freeze(
 
     ``floor`` is the eligible-commit count a repository needs before its ordinary pushes are drawn. The
     held-out benchmark keeps the shipped 40; a training freeze may lower it (never below the ratio) so small
-    repositories with 20 eligible commits still contribute, and the record carries the floor used."""
+    repositories with 20 eligible commits still contribute, and the record carries the floor used.
+    ``history_cache`` reuses completed per-commit history rows across freezes of the same clone."""
     if type(ratio) is not int or ratio < 20:
         raise ValueError("ordinary ratio must be at least 20")
     if type(floor) is not int or floor < ratio:
@@ -91,7 +93,8 @@ def freeze(
         functions = {(case["path"], case["function"]) for case in all_cases}
         head = (history_heads or {}).get(repo) or git.run("rev-parse", "HEAD").strip()
         heads[repo] = head
-        history = sorted(extract.ordinary_history(git, head), key=lambda row: row["head"])
+        history_options = {"cache_dir": history_cache} if history_cache is not None else {}
+        history = sorted(extract.ordinary_history(git, head, **history_options), key=lambda row: row["head"])
         # Ratio is per positive CHANGE (a fix commit), not per accepted function. Several
         # accepted functions in one fix collapse to one vulnerability change below, so the
         # negative count must track distinct (commit, parent) pairs.
@@ -177,6 +180,7 @@ def main(argv=None):
     parser.add_argument("--used", type=Path, required=True, help="JSON array of used repository identities, including known aliases")
     parser.add_argument("--out", type=Path, required=True)
     parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--history-cache", type=Path, help="Directory for per-clone commit history caches")
     args = parser.parse_args(argv)
     result = freeze(
         args.out,
@@ -185,6 +189,7 @@ def main(argv=None):
         repos={k: local_git(Path(v)) for k, v in json.loads(args.repos.read_text()).items()},
         used=json.loads(args.used.read_text()),
         seed=args.seed,
+        history_cache=args.history_cache,
     )
     print(json.dumps(result, sort_keys=True))
     return 0

tests/test_unseen_extract.py

@@ -1,6 +1,8 @@
 """Offline extraction: an injected local Git client over a synthetic temporary history."""
 
+import hashlib
 import importlib
+import json
 import signal
 import subprocess
 import sys
@@ -8,6 +10,7 @@ import time
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import replace
 from pathlib import Path
+from unittest.mock import Mock
 
 import pytest
 
@@ -424,6 +427,175 @@ def test_git_ordinary_collection_generated_attributes_and_headers(history):
     assert files["app.py"]["head_lines"]
 
 
+def count_history_work(git, monkeypatch):
+    calls = {}
+    for target, names in ((git, ("stats", "hunks", "blob")), (e, ("functions", "generated"))):
+        for name in names:
+            calls[name] = Mock(wraps=getattr(target, name))
+            monkeypatch.setattr(target, name, calls[name])
+    return calls
+
+
+def test_history_cache_matches_rows_and_skips_commit_work(history, monkeypatch):
+    git, commit, _, _, _, _, commands = history
+    commit("# Generated by fixture\ndef handler(value):\n    return value\n")
+    expected = e.ordinary_history(git, "HEAD")
+    calls = count_history_work(git, monkeypatch)
+    cache = git.path / ".git/history-cache"
+    assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
+    assert all(call.call_count > 0 for call in calls.values())
+    top = git.run("rev-parse", "--show-toplevel").strip()
+    (path,) = cache.iterdir()
+    assert path.name == hashlib.sha256(top.encode()).hexdigest() + ".jsonl"
+    before = path.read_bytes()
+    assert [json.loads(line) for line in before.splitlines()] == [
+        {"version": e.HISTORY_CACHE_VERSION, "head": row["head"], "row": row} for row in expected
+    ]
+    for call in calls.values():
+        call.reset_mock()
+    commands.clear()
+    assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
+    assert all(call.call_count == 0 for call in calls.values())
+    assert sum("log" in argv for argv in commands) == 1
+    assert path.read_bytes() == before
+
+
+@pytest.mark.parametrize("tail", [b'{"version": 1, "row": ', b'{"row": "\xc3'])
+def test_history_cache_resumes_after_partial_line(history, monkeypatch, tail):
+    git, _, _, _, _, _, _ = history
+    expected = e.ordinary_history(git, "HEAD")
+    cache = git.path / ".git/history-cache"
+    e.ordinary_history(git, "HEAD", cache_dir=cache)
+    (path,) = cache.iterdir()
+    count = 2
+    prefix = b"".join(path.read_bytes().splitlines(keepends=True)[:count]) + tail
+    path.write_bytes(prefix)
+    calls = count_history_work(git, monkeypatch)
+    assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
+    missing = [row for row in expected[count:] if len(row["parents"]) == 1]
+    assert [call.args[1] for call in calls["stats"].call_args_list] == [row["head"] for row in missing]
+    assert calls["hunks"].call_count > 0 and calls["blob"].call_count > 0
+    after = path.read_bytes()
+    assert after.startswith(prefix + b"\n")
+    assert [json.loads(line)["row"] for line in after.splitlines()[count + 1 :]] == expected[count:]
+    for call in calls.values():
+        call.reset_mock()
+    assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
+    assert all(call.call_count == 0 for call in calls.values())
+
+
+@pytest.mark.parametrize("invalid", ["version", "parents", "json", "head"])
+def test_history_cache_recomputes_invalid_entry(history, monkeypatch, invalid):
+    git, _, _, _, _, _, _ = history
+    expected = e.ordinary_history(git, "HEAD")
+    cache = git.path / ".git/history-cache"
+    e.ordinary_history(git, "HEAD", cache_dir=cache)
+    (path,) = cache.iterdir()
+    lines = path.read_text().splitlines(keepends=True)
+    entry = json.loads(lines[0])
+    if invalid == "version":
+        entry["version"] += 1
+    elif invalid == "parents":
+        entry["row"]["parents"] = []
+    elif invalid == "head":
+        entry["head"] = "f" * 40
+    lines[0] = "broken JSON\n" if invalid == "json" else json.dumps(entry) + "\n"
+    before = "".join(lines)
+    path.write_text(before)
+    calls = count_history_work(git, monkeypatch)
+    assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
+    calls["stats"].assert_called_once_with(expected[0]["parents"][0], expected[0]["head"])
+    assert calls["hunks"].call_count > 0 and calls["blob"].call_count > 0
+    assert path.read_text().startswith(before)
+    assert json.loads(path.read_text().splitlines()[-1])["row"] == expected[0]
+
+
+def test_history_without_cache_writes_nothing(history):
+    git, _, _, _, _, _, commands = history
+    before = {path.relative_to(git.path) for path in git.path.rglob("*")}
+    expected = e.ordinary_history(git, "HEAD")
+    assert e.ordinary_history(git, "HEAD", cache_dir=None) == expected
+    assert {path.relative_to(git.path) for path in git.path.rglob("*")} == before
+    assert not any("--show-toplevel" in argv for argv in commands)
+
+
+def test_history_cache_tracks_new_heads_and_current_log(history, monkeypatch):
+    git, commit, _, _, _, _, _ = history
+    cache = git.path / ".git/history-cache"
+    original = e.ordinary_history(git, "HEAD", cache_dir=cache)
+    head = commit("def handler(value):\n    return value + 1\n")
+    expected = e.ordinary_history(git, head)
+    calls = count_history_work(git, monkeypatch)
+    assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
+    calls["stats"].assert_called_once_with(original[0]["head"], head)
+    for call in calls.values():
+        call.reset_mock()
+    assert e.ordinary_history(git, original[0]["head"], cache_dir=cache) == original
+    assert all(call.call_count == 0 for call in calls.values())
+
+
+def test_history_cache_includes_merges_and_roots(history, monkeypatch):
+    git, _, _, root, _, _, _ = history
+    head = git.run("rev-parse", "HEAD").strip()
+    git.run("checkout", "-qb", "side", root)
+    (git.path / "side.txt").write_text("side branch\n")
+    git.run("add", "side.txt")
+    git.run("commit", "-qm", "side")
+    git.run("checkout", "-q", head)
+    git.run("merge", "--no-ff", "-qm", "merge", "side")
+    expected = e.ordinary_history(git, "HEAD")
+    assert len(expected[0]["parents"]) == 2 and expected[0]["files"] == []
+    assert expected[-1]["parents"] == [] and expected[-1]["files"] == []
+    cache = git.path / ".git/history-cache"
+    assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
+    (path,) = cache.iterdir()
+    assert [json.loads(line)["row"] for line in path.read_text().splitlines()] == expected
+    calls = count_history_work(git, monkeypatch)
+    assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
+    assert all(call.call_count == 0 for call in calls.values())
+
+
+def test_history_cache_is_per_clone_and_uses_top_level(history, monkeypatch):
+    git, _, _, _, _, _, _ = history
+    cache = git.path / ".git/history-cache"
+    expected = e.ordinary_history(git, "HEAD", cache_dir=cache)
+    nested = git.path / "nested"
+    nested.mkdir()
+    subdir_git = e.Git(nested)
+    calls = count_history_work(subdir_git, monkeypatch)
+    assert e.ordinary_history(subdir_git, "HEAD", cache_dir=cache) == expected
+    assert all(call.call_count == 0 for call in calls.values())
+    clone_path = git.path / "clone"
+    subprocess.run(["git", "clone", "--quiet", str(git.path), str(clone_path)], check=True, capture_output=True)
+    clone = e.Git(clone_path)
+    calls = count_history_work(clone, monkeypatch)
+    assert e.ordinary_history(clone, "HEAD", cache_dir=cache) == expected
+    assert all(call.call_count > 0 for call in calls.values())
+    assert len(list(cache.glob("*.jsonl"))) == 2
+
+
+def test_history_cache_flushes_each_completed_commit(history, monkeypatch):
+    git, _, _, _, _, _, _ = history
+    expected = e.ordinary_history(git, "HEAD")
+    cache = git.path / ".git/history-cache"
+    stats = git.stats
+
+    def interrupted(base, head):
+        if head == expected[1]["head"]:
+            (path,) = cache.iterdir()
+            assert [json.loads(line)["row"] for line in path.read_text().splitlines()] == expected[:1]
+            raise e.ExtractionTimeout("interrupted")
+        return stats(base, head)
+
+    monkeypatch.setattr(git, "stats", interrupted)
+    with pytest.raises(e.ExtractionTimeout, match="interrupted"):
+        e.ordinary_history(git, "HEAD", cache_dir=cache)
+    monkeypatch.setattr(git, "stats", stats)
+    calls = count_history_work(git, monkeypatch)
+    assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
+    assert expected[0]["head"] not in [call.args[1] for call in calls["stats"].call_args_list]
+
+
 def test_attribute_unset_and_mixed_generated_source(history):
     git, commit, _, _, _, _, _ = history
     sha = commit("x = 1\n", extra={".gitattributes": "*.py linguist-generated\napp.py -linguist-generated\n"})

tests/test_valid_benchmark.py

@@ -366,15 +366,70 @@ def test_empty_entry_point_cannot_accept(pair):
     assert verify.spot_check(verify.verify_case(git, case, client=stub), "accept", "AB", "2026-10-12")["verdict"] == "rejected"
 
 
-def test_real_local_history_freezes_without_network(pair, tmp_path):
+@pytest.mark.parametrize("use_cache", [False, True])
+def test_real_local_history_freezes_without_network(pair, tmp_path, monkeypatch, use_cache):
     git, case = pair
     for i in range(55):
         (git.path / "other.py").write_text(f"def other():\n    return {i}\n")
         git.run("add", "other.py")
         git.run("commit", "-qm", "ordinary edit")
-    result = assemble.freeze(tmp_path / "freeze", cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
+    kwargs = dict(cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
+    expected = assemble.freeze(tmp_path / "uncached", **kwargs)
+    cache = tmp_path / "history-cache" if use_cache else None
+    result = assemble.freeze(tmp_path / "freeze", **kwargs, history_cache=cache)
+    assert result == expected
     assert result["label_check_counts"]["confirmed"] == 1
     assert len(replay.load_slice(tmp_path / "freeze/pool-valid.toml", 1)) == 21
+    if use_cache:
+        (path,) = cache.glob("*.jsonl")
+        before = path.read_bytes()
+        assert len(before.splitlines()) == 57
+
+        def forbidden(*args):
+            pytest.fail("cached freeze repeated per-commit work")
+
+        for name in ("stats", "hunks", "blob"):
+            monkeypatch.setattr(git, name, forbidden)
+        assert assemble.freeze(tmp_path / "freeze", **kwargs, history_cache=cache) == result
+        assert path.read_bytes() == before
+
+
+def test_freeze_cli_passes_history_cache(pair, tmp_path, monkeypatch, capsys):
+    git, case = pair
+    catalog_path = tmp_path / "catalog.toml"
+    catalog.write_catalog(catalog_path, [case])
+    verification_path = tmp_path / "verification.json"
+    verification_path.write_text(json.dumps({"cases": [accepted(case)]}))
+    repos_path = tmp_path / "repos.json"
+    repos_path.write_text(json.dumps({case["repo"]: str(git.path)}))
+    used_path = tmp_path / "used.json"
+    used_path.write_text("[]")
+    cache = tmp_path / "history-cache"
+
+    def freeze(directory, **kwargs):
+        assert directory == tmp_path / "out"
+        assert kwargs["history_cache"] == cache
+        return {"checked": True}
+
+    monkeypatch.setattr(assemble, "freeze", freeze)
+    assert (
+        assemble.main(
+            [
+                str(catalog_path),
+                str(verification_path),
+                "--repos",
+                str(repos_path),
+                "--used",
+                str(used_path),
+                "--out",
+                str(tmp_path / "out"),
+                "--history-cache",
+                str(cache),
+            ]
+        )
+        == 0
+    )
+    assert json.loads(capsys.readouterr().out) == {"checked": True}
 
 
 def test_insufficient_negatives_and_invalid_ratio(pair, tmp_path, monkeypatch):