From 47d0f0fac1717cc3ad173e0eee7a1620f414e087 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:40:34 +0200 Subject: [PATCH] bench(unseen): per-commit history cache for ordinary_history; assemble.freeze --history-cache One append-only JSONL per clone (named by the sha256 of its top-level path), one line per commit with the row as the walk built it and HISTORY_CACHE_VERSION, flushed after every commit so a killed run keeps its progress. A cached row is reused only when its parents match the current log; other versions and torn lines are recomputed. With no cache directory nothing changes. The 7,400-commit trigger.dev walk that took over an hour per freeze becomes a one-time cost. Implemented by Codex (gpt-6-astra), gated here. Co-Authored-By: Claude Fable 5.1 --- benchmarks/unseen/extract.py | 116 +++++++++++++++-------- benchmarks/valid/assemble.py | 9 +- tests/test_unseen_extract.py | 172 ++++++++++++++++++++++++++++++++++ tests/test_valid_benchmark.py | 59 +++++++++++- 4 files changed, 315 insertions(+), 41 deletions(-) diff --git a/benchmarks/unseen/extract.py b/benchmarks/unseen/extract.py index 0bdc101..edd453e 100644 --- a/benchmarks/unseen/extract.py +++ b/benchmarks/unseen/extract.py @@ -8,6 +8,7 @@ are dropped, and the recorded SZZ bias remains subject to task 2.6's label check from __future__ import annotations import ast +import hashlib import json import re import signal @@ -23,6 +24,8 @@ from .source import Candidate, Rejected EXTRACTION_CONTRACT = "v1" # Bump only when enumeration, eligibility or source.candidate outputs change. RESOLUTION_CONTRACT = "v1" +# Bump whenever functions(), generated(), LANGUAGES or the row shape change: cached rows embed their results. +HISTORY_CACHE_VERSION = 1 LANGUAGES = {".py", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".php", ".java"} HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@", re.M) @@ -452,45 +455,84 @@ def generated(git, revision: str, path: str, text: str) -> bool: return result -def ordinary_history(git, head: str) -> list[dict]: +@contextmanager +def _history_cache(git, cache_dir: Path | None): + """Load completed rows and open the clone's cache for appending.""" + if cache_dir is None: + yield {}, None + return + top = git.run("rev-parse", "--show-toplevel").strip() + cache_dir.mkdir(parents=True, exist_ok=True) + path = cache_dir / (hashlib.sha256(top.encode()).hexdigest() + ".jsonl") + with path.open("a+b") as handle: + handle.seek(0) + cached = {} + line = b"" + for line in handle: + try: + entry = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + continue + if not isinstance(entry, dict) or entry.get("version") != HISTORY_CACHE_VERSION: + continue + row = entry.get("row") + sha = entry.get("head") + if isinstance(sha, str) and isinstance(row, dict) and row.get("head") == sha: + cached[sha] = row + if line and not line.endswith(b"\n"): + # Keep a partial last line from swallowing the next completed row. + handle.write(b"\n") + handle.flush() + yield cached, handle + + +def ordinary_history(git, head: str, *, cache_dir: Path | None = None) -> list[dict]: + """Read first-parent history, optionally reusing completed rows from this clone.""" raw = git.run("log", "--first-parent", "--format=%H%x00%P%x00%ct%x00%an%x00%B%x1e", head) rows = [] - for entry in raw.split("\x1e"): - if not entry.strip(): - continue - fields = entry.strip().split("\x00") - if len(fields) != 5 or not SHA.fullmatch(fields[0]): - raise InstrumentFailure("invalid_history") - sha, parents, timestamp, author, message = fields - row = {"head": sha, "parents": parents.split(), "timestamp": int(timestamp), "author": author, "message": message, "files": []} - rows.append(row) - if len(row["parents"]) != 1: - continue - base = row["parents"][0] - for path, added, removed in git.stats(base, sha): - hunks = git.hunks(base, sha, path) - info = { - "path": path, - "added": added, - "removed": removed, - "head_lines": [[max(1, n), max(1, n + max(c, 1) - 1)] for _, _, n, c in hunks], - "functions": [], - "generated": False, - } - if Path(path).suffix in LANGUAGES: - # Inspect both sides: deleting a vulnerable function is still touching it. - for revision, side in ((base, 0), (sha, 2)): - text = git.blob(revision, path) - if text is None: - continue - spans = functions(path, text) - for hunk in hunks: - start, count = hunk[side : side + 2] - for name, lo, hi in spans: - if max(1, start) <= hi and start + max(count, 1) - 1 >= lo: - info["functions"].append(name) - info["generated"] |= generated(git, revision, path, text) - row["files"].append(info) + with _history_cache(git, cache_dir) as (cached, handle): + for entry in raw.split("\x1e"): + if not entry.strip(): + continue + fields = entry.strip().split("\x00") + if len(fields) != 5 or not SHA.fullmatch(fields[0]): + raise InstrumentFailure("invalid_history") + sha, parents, timestamp, author, message = fields + previous = cached.get(sha) + if previous is not None and previous.get("parents") == parents.split(): + rows.append(previous) + continue + row = {"head": sha, "parents": parents.split(), "timestamp": int(timestamp), "author": author, "message": message, "files": []} + rows.append(row) + if len(row["parents"]) == 1: + base = row["parents"][0] + for path, added, removed in git.stats(base, sha): + hunks = git.hunks(base, sha, path) + info = { + "path": path, + "added": added, + "removed": removed, + "head_lines": [[max(1, n), max(1, n + max(c, 1) - 1)] for _, _, n, c in hunks], + "functions": [], + "generated": False, + } + if Path(path).suffix in LANGUAGES: + # Inspect both sides: deleting a vulnerable function is still touching it. + for revision, side in ((base, 0), (sha, 2)): + text = git.blob(revision, path) + if text is None: + continue + spans = functions(path, text) + for hunk in hunks: + start, count = hunk[side : side + 2] + for name, lo, hi in spans: + if max(1, start) <= hi and start + max(count, 1) - 1 >= lo: + info["functions"].append(name) + info["generated"] |= generated(git, revision, path, text) + row["files"].append(info) + if handle is not None: + handle.write((json.dumps({"version": HISTORY_CACHE_VERSION, "head": sha, "row": row}) + "\n").encode()) + handle.flush() return rows diff --git a/benchmarks/valid/assemble.py b/benchmarks/valid/assemble.py index efb0a9f..dc7179b 100644 --- a/benchmarks/valid/assemble.py +++ b/benchmarks/valid/assemble.py @@ -51,6 +51,7 @@ def freeze( seed: int = 0, ratio: int = 20, history_heads: dict | None = None, + history_cache: Path | None = None, fix_index: dict | None = None, floor: int = extract.ORDINARY_FLOOR, ) -> dict: @@ -58,7 +59,8 @@ def freeze( ``floor`` is the eligible-commit count a repository needs before its ordinary pushes are drawn. The held-out benchmark keeps the shipped 40; a training freeze may lower it (never below the ratio) so small - repositories with 20 eligible commits still contribute, and the record carries the floor used.""" + repositories with 20 eligible commits still contribute, and the record carries the floor used. + ``history_cache`` reuses completed per-commit history rows across freezes of the same clone.""" if type(ratio) is not int or ratio < 20: raise ValueError("ordinary ratio must be at least 20") if type(floor) is not int or floor < ratio: @@ -91,7 +93,8 @@ def freeze( functions = {(case["path"], case["function"]) for case in all_cases} head = (history_heads or {}).get(repo) or git.run("rev-parse", "HEAD").strip() heads[repo] = head - history = sorted(extract.ordinary_history(git, head), key=lambda row: row["head"]) + history_options = {"cache_dir": history_cache} if history_cache is not None else {} + history = sorted(extract.ordinary_history(git, head, **history_options), key=lambda row: row["head"]) # Ratio is per positive CHANGE (a fix commit), not per accepted function. Several # accepted functions in one fix collapse to one vulnerability change below, so the # negative count must track distinct (commit, parent) pairs. @@ -177,6 +180,7 @@ def main(argv=None): parser.add_argument("--used", type=Path, required=True, help="JSON array of used repository identities, including known aliases") parser.add_argument("--out", type=Path, required=True) parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--history-cache", type=Path, help="Directory for per-clone commit history caches") args = parser.parse_args(argv) result = freeze( args.out, @@ -185,6 +189,7 @@ def main(argv=None): repos={k: local_git(Path(v)) for k, v in json.loads(args.repos.read_text()).items()}, used=json.loads(args.used.read_text()), seed=args.seed, + history_cache=args.history_cache, ) print(json.dumps(result, sort_keys=True)) return 0 diff --git a/tests/test_unseen_extract.py b/tests/test_unseen_extract.py index 9a1fbad..d2f2f92 100644 --- a/tests/test_unseen_extract.py +++ b/tests/test_unseen_extract.py @@ -1,6 +1,8 @@ """Offline extraction: an injected local Git client over a synthetic temporary history.""" +import hashlib import importlib +import json import signal import subprocess import sys @@ -8,6 +10,7 @@ import time from concurrent.futures import ThreadPoolExecutor from dataclasses import replace from pathlib import Path +from unittest.mock import Mock import pytest @@ -424,6 +427,175 @@ def test_git_ordinary_collection_generated_attributes_and_headers(history): assert files["app.py"]["head_lines"] +def count_history_work(git, monkeypatch): + calls = {} + for target, names in ((git, ("stats", "hunks", "blob")), (e, ("functions", "generated"))): + for name in names: + calls[name] = Mock(wraps=getattr(target, name)) + monkeypatch.setattr(target, name, calls[name]) + return calls + + +def test_history_cache_matches_rows_and_skips_commit_work(history, monkeypatch): + git, commit, _, _, _, _, commands = history + commit("# Generated by fixture\ndef handler(value):\n return value\n") + expected = e.ordinary_history(git, "HEAD") + calls = count_history_work(git, monkeypatch) + cache = git.path / ".git/history-cache" + assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected + assert all(call.call_count > 0 for call in calls.values()) + top = git.run("rev-parse", "--show-toplevel").strip() + (path,) = cache.iterdir() + assert path.name == hashlib.sha256(top.encode()).hexdigest() + ".jsonl" + before = path.read_bytes() + assert [json.loads(line) for line in before.splitlines()] == [ + {"version": e.HISTORY_CACHE_VERSION, "head": row["head"], "row": row} for row in expected + ] + for call in calls.values(): + call.reset_mock() + commands.clear() + assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected + assert all(call.call_count == 0 for call in calls.values()) + assert sum("log" in argv for argv in commands) == 1 + assert path.read_bytes() == before + + +@pytest.mark.parametrize("tail", [b'{"version": 1, "row": ', b'{"row": "\xc3']) +def test_history_cache_resumes_after_partial_line(history, monkeypatch, tail): + git, _, _, _, _, _, _ = history + expected = e.ordinary_history(git, "HEAD") + cache = git.path / ".git/history-cache" + e.ordinary_history(git, "HEAD", cache_dir=cache) + (path,) = cache.iterdir() + count = 2 + prefix = b"".join(path.read_bytes().splitlines(keepends=True)[:count]) + tail + path.write_bytes(prefix) + calls = count_history_work(git, monkeypatch) + assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected + missing = [row for row in expected[count:] if len(row["parents"]) == 1] + assert [call.args[1] for call in calls["stats"].call_args_list] == [row["head"] for row in missing] + assert calls["hunks"].call_count > 0 and calls["blob"].call_count > 0 + after = path.read_bytes() + assert after.startswith(prefix + b"\n") + assert [json.loads(line)["row"] for line in after.splitlines()[count + 1 :]] == expected[count:] + for call in calls.values(): + call.reset_mock() + assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected + assert all(call.call_count == 0 for call in calls.values()) + + +@pytest.mark.parametrize("invalid", ["version", "parents", "json", "head"]) +def test_history_cache_recomputes_invalid_entry(history, monkeypatch, invalid): + git, _, _, _, _, _, _ = history + expected = e.ordinary_history(git, "HEAD") + cache = git.path / ".git/history-cache" + e.ordinary_history(git, "HEAD", cache_dir=cache) + (path,) = cache.iterdir() + lines = path.read_text().splitlines(keepends=True) + entry = json.loads(lines[0]) + if invalid == "version": + entry["version"] += 1 + elif invalid == "parents": + entry["row"]["parents"] = [] + elif invalid == "head": + entry["head"] = "f" * 40 + lines[0] = "broken JSON\n" if invalid == "json" else json.dumps(entry) + "\n" + before = "".join(lines) + path.write_text(before) + calls = count_history_work(git, monkeypatch) + assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected + calls["stats"].assert_called_once_with(expected[0]["parents"][0], expected[0]["head"]) + assert calls["hunks"].call_count > 0 and calls["blob"].call_count > 0 + assert path.read_text().startswith(before) + assert json.loads(path.read_text().splitlines()[-1])["row"] == expected[0] + + +def test_history_without_cache_writes_nothing(history): + git, _, _, _, _, _, commands = history + before = {path.relative_to(git.path) for path in git.path.rglob("*")} + expected = e.ordinary_history(git, "HEAD") + assert e.ordinary_history(git, "HEAD", cache_dir=None) == expected + assert {path.relative_to(git.path) for path in git.path.rglob("*")} == before + assert not any("--show-toplevel" in argv for argv in commands) + + +def test_history_cache_tracks_new_heads_and_current_log(history, monkeypatch): + git, commit, _, _, _, _, _ = history + cache = git.path / ".git/history-cache" + original = e.ordinary_history(git, "HEAD", cache_dir=cache) + head = commit("def handler(value):\n return value + 1\n") + expected = e.ordinary_history(git, head) + calls = count_history_work(git, monkeypatch) + assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected + calls["stats"].assert_called_once_with(original[0]["head"], head) + for call in calls.values(): + call.reset_mock() + assert e.ordinary_history(git, original[0]["head"], cache_dir=cache) == original + assert all(call.call_count == 0 for call in calls.values()) + + +def test_history_cache_includes_merges_and_roots(history, monkeypatch): + git, _, _, root, _, _, _ = history + head = git.run("rev-parse", "HEAD").strip() + git.run("checkout", "-qb", "side", root) + (git.path / "side.txt").write_text("side branch\n") + git.run("add", "side.txt") + git.run("commit", "-qm", "side") + git.run("checkout", "-q", head) + git.run("merge", "--no-ff", "-qm", "merge", "side") + expected = e.ordinary_history(git, "HEAD") + assert len(expected[0]["parents"]) == 2 and expected[0]["files"] == [] + assert expected[-1]["parents"] == [] and expected[-1]["files"] == [] + cache = git.path / ".git/history-cache" + assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected + (path,) = cache.iterdir() + assert [json.loads(line)["row"] for line in path.read_text().splitlines()] == expected + calls = count_history_work(git, monkeypatch) + assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected + assert all(call.call_count == 0 for call in calls.values()) + + +def test_history_cache_is_per_clone_and_uses_top_level(history, monkeypatch): + git, _, _, _, _, _, _ = history + cache = git.path / ".git/history-cache" + expected = e.ordinary_history(git, "HEAD", cache_dir=cache) + nested = git.path / "nested" + nested.mkdir() + subdir_git = e.Git(nested) + calls = count_history_work(subdir_git, monkeypatch) + assert e.ordinary_history(subdir_git, "HEAD", cache_dir=cache) == expected + assert all(call.call_count == 0 for call in calls.values()) + clone_path = git.path / "clone" + subprocess.run(["git", "clone", "--quiet", str(git.path), str(clone_path)], check=True, capture_output=True) + clone = e.Git(clone_path) + calls = count_history_work(clone, monkeypatch) + assert e.ordinary_history(clone, "HEAD", cache_dir=cache) == expected + assert all(call.call_count > 0 for call in calls.values()) + assert len(list(cache.glob("*.jsonl"))) == 2 + + +def test_history_cache_flushes_each_completed_commit(history, monkeypatch): + git, _, _, _, _, _, _ = history + expected = e.ordinary_history(git, "HEAD") + cache = git.path / ".git/history-cache" + stats = git.stats + + def interrupted(base, head): + if head == expected[1]["head"]: + (path,) = cache.iterdir() + assert [json.loads(line)["row"] for line in path.read_text().splitlines()] == expected[:1] + raise e.ExtractionTimeout("interrupted") + return stats(base, head) + + monkeypatch.setattr(git, "stats", interrupted) + with pytest.raises(e.ExtractionTimeout, match="interrupted"): + e.ordinary_history(git, "HEAD", cache_dir=cache) + monkeypatch.setattr(git, "stats", stats) + calls = count_history_work(git, monkeypatch) + assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected + assert expected[0]["head"] not in [call.args[1] for call in calls["stats"].call_args_list] + + def test_attribute_unset_and_mixed_generated_source(history): git, commit, _, _, _, _, _ = history sha = commit("x = 1\n", extra={".gitattributes": "*.py linguist-generated\napp.py -linguist-generated\n"}) diff --git a/tests/test_valid_benchmark.py b/tests/test_valid_benchmark.py index 91a546d..4007955 100644 --- a/tests/test_valid_benchmark.py +++ b/tests/test_valid_benchmark.py @@ -366,15 +366,70 @@ def test_empty_entry_point_cannot_accept(pair): assert verify.spot_check(verify.verify_case(git, case, client=stub), "accept", "AB", "2026-10-12")["verdict"] == "rejected" -def test_real_local_history_freezes_without_network(pair, tmp_path): +@pytest.mark.parametrize("use_cache", [False, True]) +def test_real_local_history_freezes_without_network(pair, tmp_path, monkeypatch, use_cache): git, case = pair for i in range(55): (git.path / "other.py").write_text(f"def other():\n return {i}\n") git.run("add", "other.py") git.run("commit", "-qm", "ordinary edit") - result = assemble.freeze(tmp_path / "freeze", cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set()) + kwargs = dict(cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set()) + expected = assemble.freeze(tmp_path / "uncached", **kwargs) + cache = tmp_path / "history-cache" if use_cache else None + result = assemble.freeze(tmp_path / "freeze", **kwargs, history_cache=cache) + assert result == expected assert result["label_check_counts"]["confirmed"] == 1 assert len(replay.load_slice(tmp_path / "freeze/pool-valid.toml", 1)) == 21 + if use_cache: + (path,) = cache.glob("*.jsonl") + before = path.read_bytes() + assert len(before.splitlines()) == 57 + + def forbidden(*args): + pytest.fail("cached freeze repeated per-commit work") + + for name in ("stats", "hunks", "blob"): + monkeypatch.setattr(git, name, forbidden) + assert assemble.freeze(tmp_path / "freeze", **kwargs, history_cache=cache) == result + assert path.read_bytes() == before + + +def test_freeze_cli_passes_history_cache(pair, tmp_path, monkeypatch, capsys): + git, case = pair + catalog_path = tmp_path / "catalog.toml" + catalog.write_catalog(catalog_path, [case]) + verification_path = tmp_path / "verification.json" + verification_path.write_text(json.dumps({"cases": [accepted(case)]})) + repos_path = tmp_path / "repos.json" + repos_path.write_text(json.dumps({case["repo"]: str(git.path)})) + used_path = tmp_path / "used.json" + used_path.write_text("[]") + cache = tmp_path / "history-cache" + + def freeze(directory, **kwargs): + assert directory == tmp_path / "out" + assert kwargs["history_cache"] == cache + return {"checked": True} + + monkeypatch.setattr(assemble, "freeze", freeze) + assert ( + assemble.main( + [ + str(catalog_path), + str(verification_path), + "--repos", + str(repos_path), + "--used", + str(used_path), + "--out", + str(tmp_path / "out"), + "--history-cache", + str(cache), + ] + ) + == 0 + ) + assert json.loads(capsys.readouterr().out) == {"checked": True} def test_insufficient_negatives_and_invalid_ratio(pair, tmp_path, monkeypatch):