mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 12:09:27 +00:00
bench(unseen): per-commit history cache for ordinary_history; assemble.freeze --history-cache
One append-only JSONL per clone (named by the sha256 of its top-level path), one line per commit with the row as the walk built it and HISTORY_CACHE_VERSION, flushed after every commit so a killed run keeps its progress. A cached row is reused only when its parents match the current log; other versions and torn lines are recomputed. With no cache directory nothing changes. The 7,400-commit trigger.dev walk that took over an hour per freeze becomes a one-time cost. Implemented by Codex (gpt-6-astra), gated here. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
dbaaa86d5b
commit
47d0f0fac1
@@ -1,6 +1,8 @@
|
||||
"""Offline extraction: an injected local Git client over a synthetic temporary history."""
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -8,6 +10,7 @@ import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -424,6 +427,175 @@ def test_git_ordinary_collection_generated_attributes_and_headers(history):
|
||||
assert files["app.py"]["head_lines"]
|
||||
|
||||
|
||||
def count_history_work(git, monkeypatch):
|
||||
calls = {}
|
||||
for target, names in ((git, ("stats", "hunks", "blob")), (e, ("functions", "generated"))):
|
||||
for name in names:
|
||||
calls[name] = Mock(wraps=getattr(target, name))
|
||||
monkeypatch.setattr(target, name, calls[name])
|
||||
return calls
|
||||
|
||||
|
||||
def test_history_cache_matches_rows_and_skips_commit_work(history, monkeypatch):
|
||||
git, commit, _, _, _, _, commands = history
|
||||
commit("# Generated by fixture\ndef handler(value):\n return value\n")
|
||||
expected = e.ordinary_history(git, "HEAD")
|
||||
calls = count_history_work(git, monkeypatch)
|
||||
cache = git.path / ".git/history-cache"
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
|
||||
assert all(call.call_count > 0 for call in calls.values())
|
||||
top = git.run("rev-parse", "--show-toplevel").strip()
|
||||
(path,) = cache.iterdir()
|
||||
assert path.name == hashlib.sha256(top.encode()).hexdigest() + ".jsonl"
|
||||
before = path.read_bytes()
|
||||
assert [json.loads(line) for line in before.splitlines()] == [
|
||||
{"version": e.HISTORY_CACHE_VERSION, "head": row["head"], "row": row} for row in expected
|
||||
]
|
||||
for call in calls.values():
|
||||
call.reset_mock()
|
||||
commands.clear()
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
|
||||
assert all(call.call_count == 0 for call in calls.values())
|
||||
assert sum("log" in argv for argv in commands) == 1
|
||||
assert path.read_bytes() == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tail", [b'{"version": 1, "row": ', b'{"row": "\xc3'])
|
||||
def test_history_cache_resumes_after_partial_line(history, monkeypatch, tail):
|
||||
git, _, _, _, _, _, _ = history
|
||||
expected = e.ordinary_history(git, "HEAD")
|
||||
cache = git.path / ".git/history-cache"
|
||||
e.ordinary_history(git, "HEAD", cache_dir=cache)
|
||||
(path,) = cache.iterdir()
|
||||
count = 2
|
||||
prefix = b"".join(path.read_bytes().splitlines(keepends=True)[:count]) + tail
|
||||
path.write_bytes(prefix)
|
||||
calls = count_history_work(git, monkeypatch)
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
|
||||
missing = [row for row in expected[count:] if len(row["parents"]) == 1]
|
||||
assert [call.args[1] for call in calls["stats"].call_args_list] == [row["head"] for row in missing]
|
||||
assert calls["hunks"].call_count > 0 and calls["blob"].call_count > 0
|
||||
after = path.read_bytes()
|
||||
assert after.startswith(prefix + b"\n")
|
||||
assert [json.loads(line)["row"] for line in after.splitlines()[count + 1 :]] == expected[count:]
|
||||
for call in calls.values():
|
||||
call.reset_mock()
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
|
||||
assert all(call.call_count == 0 for call in calls.values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid", ["version", "parents", "json", "head"])
|
||||
def test_history_cache_recomputes_invalid_entry(history, monkeypatch, invalid):
|
||||
git, _, _, _, _, _, _ = history
|
||||
expected = e.ordinary_history(git, "HEAD")
|
||||
cache = git.path / ".git/history-cache"
|
||||
e.ordinary_history(git, "HEAD", cache_dir=cache)
|
||||
(path,) = cache.iterdir()
|
||||
lines = path.read_text().splitlines(keepends=True)
|
||||
entry = json.loads(lines[0])
|
||||
if invalid == "version":
|
||||
entry["version"] += 1
|
||||
elif invalid == "parents":
|
||||
entry["row"]["parents"] = []
|
||||
elif invalid == "head":
|
||||
entry["head"] = "f" * 40
|
||||
lines[0] = "broken JSON\n" if invalid == "json" else json.dumps(entry) + "\n"
|
||||
before = "".join(lines)
|
||||
path.write_text(before)
|
||||
calls = count_history_work(git, monkeypatch)
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
|
||||
calls["stats"].assert_called_once_with(expected[0]["parents"][0], expected[0]["head"])
|
||||
assert calls["hunks"].call_count > 0 and calls["blob"].call_count > 0
|
||||
assert path.read_text().startswith(before)
|
||||
assert json.loads(path.read_text().splitlines()[-1])["row"] == expected[0]
|
||||
|
||||
|
||||
def test_history_without_cache_writes_nothing(history):
|
||||
git, _, _, _, _, _, commands = history
|
||||
before = {path.relative_to(git.path) for path in git.path.rglob("*")}
|
||||
expected = e.ordinary_history(git, "HEAD")
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=None) == expected
|
||||
assert {path.relative_to(git.path) for path in git.path.rglob("*")} == before
|
||||
assert not any("--show-toplevel" in argv for argv in commands)
|
||||
|
||||
|
||||
def test_history_cache_tracks_new_heads_and_current_log(history, monkeypatch):
|
||||
git, commit, _, _, _, _, _ = history
|
||||
cache = git.path / ".git/history-cache"
|
||||
original = e.ordinary_history(git, "HEAD", cache_dir=cache)
|
||||
head = commit("def handler(value):\n return value + 1\n")
|
||||
expected = e.ordinary_history(git, head)
|
||||
calls = count_history_work(git, monkeypatch)
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
|
||||
calls["stats"].assert_called_once_with(original[0]["head"], head)
|
||||
for call in calls.values():
|
||||
call.reset_mock()
|
||||
assert e.ordinary_history(git, original[0]["head"], cache_dir=cache) == original
|
||||
assert all(call.call_count == 0 for call in calls.values())
|
||||
|
||||
|
||||
def test_history_cache_includes_merges_and_roots(history, monkeypatch):
|
||||
git, _, _, root, _, _, _ = history
|
||||
head = git.run("rev-parse", "HEAD").strip()
|
||||
git.run("checkout", "-qb", "side", root)
|
||||
(git.path / "side.txt").write_text("side branch\n")
|
||||
git.run("add", "side.txt")
|
||||
git.run("commit", "-qm", "side")
|
||||
git.run("checkout", "-q", head)
|
||||
git.run("merge", "--no-ff", "-qm", "merge", "side")
|
||||
expected = e.ordinary_history(git, "HEAD")
|
||||
assert len(expected[0]["parents"]) == 2 and expected[0]["files"] == []
|
||||
assert expected[-1]["parents"] == [] and expected[-1]["files"] == []
|
||||
cache = git.path / ".git/history-cache"
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
|
||||
(path,) = cache.iterdir()
|
||||
assert [json.loads(line)["row"] for line in path.read_text().splitlines()] == expected
|
||||
calls = count_history_work(git, monkeypatch)
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
|
||||
assert all(call.call_count == 0 for call in calls.values())
|
||||
|
||||
|
||||
def test_history_cache_is_per_clone_and_uses_top_level(history, monkeypatch):
|
||||
git, _, _, _, _, _, _ = history
|
||||
cache = git.path / ".git/history-cache"
|
||||
expected = e.ordinary_history(git, "HEAD", cache_dir=cache)
|
||||
nested = git.path / "nested"
|
||||
nested.mkdir()
|
||||
subdir_git = e.Git(nested)
|
||||
calls = count_history_work(subdir_git, monkeypatch)
|
||||
assert e.ordinary_history(subdir_git, "HEAD", cache_dir=cache) == expected
|
||||
assert all(call.call_count == 0 for call in calls.values())
|
||||
clone_path = git.path / "clone"
|
||||
subprocess.run(["git", "clone", "--quiet", str(git.path), str(clone_path)], check=True, capture_output=True)
|
||||
clone = e.Git(clone_path)
|
||||
calls = count_history_work(clone, monkeypatch)
|
||||
assert e.ordinary_history(clone, "HEAD", cache_dir=cache) == expected
|
||||
assert all(call.call_count > 0 for call in calls.values())
|
||||
assert len(list(cache.glob("*.jsonl"))) == 2
|
||||
|
||||
|
||||
def test_history_cache_flushes_each_completed_commit(history, monkeypatch):
|
||||
git, _, _, _, _, _, _ = history
|
||||
expected = e.ordinary_history(git, "HEAD")
|
||||
cache = git.path / ".git/history-cache"
|
||||
stats = git.stats
|
||||
|
||||
def interrupted(base, head):
|
||||
if head == expected[1]["head"]:
|
||||
(path,) = cache.iterdir()
|
||||
assert [json.loads(line)["row"] for line in path.read_text().splitlines()] == expected[:1]
|
||||
raise e.ExtractionTimeout("interrupted")
|
||||
return stats(base, head)
|
||||
|
||||
monkeypatch.setattr(git, "stats", interrupted)
|
||||
with pytest.raises(e.ExtractionTimeout, match="interrupted"):
|
||||
e.ordinary_history(git, "HEAD", cache_dir=cache)
|
||||
monkeypatch.setattr(git, "stats", stats)
|
||||
calls = count_history_work(git, monkeypatch)
|
||||
assert e.ordinary_history(git, "HEAD", cache_dir=cache) == expected
|
||||
assert expected[0]["head"] not in [call.args[1] for call in calls["stats"].call_args_list]
|
||||
|
||||
|
||||
def test_attribute_unset_and_mixed_generated_source(history):
|
||||
git, commit, _, _, _, _, _ = history
|
||||
sha = commit("x = 1\n", extra={".gitattributes": "*.py linguist-generated\napp.py -linguist-generated\n"})
|
||||
|
||||
@@ -366,15 +366,70 @@ def test_empty_entry_point_cannot_accept(pair):
|
||||
assert verify.spot_check(verify.verify_case(git, case, client=stub), "accept", "AB", "2026-10-12")["verdict"] == "rejected"
|
||||
|
||||
|
||||
def test_real_local_history_freezes_without_network(pair, tmp_path):
|
||||
@pytest.mark.parametrize("use_cache", [False, True])
|
||||
def test_real_local_history_freezes_without_network(pair, tmp_path, monkeypatch, use_cache):
|
||||
git, case = pair
|
||||
for i in range(55):
|
||||
(git.path / "other.py").write_text(f"def other():\n return {i}\n")
|
||||
git.run("add", "other.py")
|
||||
git.run("commit", "-qm", "ordinary edit")
|
||||
result = assemble.freeze(tmp_path / "freeze", cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
|
||||
kwargs = dict(cases=[case], verifications=[accepted(case)], repos={case["repo"]: git}, used=set())
|
||||
expected = assemble.freeze(tmp_path / "uncached", **kwargs)
|
||||
cache = tmp_path / "history-cache" if use_cache else None
|
||||
result = assemble.freeze(tmp_path / "freeze", **kwargs, history_cache=cache)
|
||||
assert result == expected
|
||||
assert result["label_check_counts"]["confirmed"] == 1
|
||||
assert len(replay.load_slice(tmp_path / "freeze/pool-valid.toml", 1)) == 21
|
||||
if use_cache:
|
||||
(path,) = cache.glob("*.jsonl")
|
||||
before = path.read_bytes()
|
||||
assert len(before.splitlines()) == 57
|
||||
|
||||
def forbidden(*args):
|
||||
pytest.fail("cached freeze repeated per-commit work")
|
||||
|
||||
for name in ("stats", "hunks", "blob"):
|
||||
monkeypatch.setattr(git, name, forbidden)
|
||||
assert assemble.freeze(tmp_path / "freeze", **kwargs, history_cache=cache) == result
|
||||
assert path.read_bytes() == before
|
||||
|
||||
|
||||
def test_freeze_cli_passes_history_cache(pair, tmp_path, monkeypatch, capsys):
|
||||
git, case = pair
|
||||
catalog_path = tmp_path / "catalog.toml"
|
||||
catalog.write_catalog(catalog_path, [case])
|
||||
verification_path = tmp_path / "verification.json"
|
||||
verification_path.write_text(json.dumps({"cases": [accepted(case)]}))
|
||||
repos_path = tmp_path / "repos.json"
|
||||
repos_path.write_text(json.dumps({case["repo"]: str(git.path)}))
|
||||
used_path = tmp_path / "used.json"
|
||||
used_path.write_text("[]")
|
||||
cache = tmp_path / "history-cache"
|
||||
|
||||
def freeze(directory, **kwargs):
|
||||
assert directory == tmp_path / "out"
|
||||
assert kwargs["history_cache"] == cache
|
||||
return {"checked": True}
|
||||
|
||||
monkeypatch.setattr(assemble, "freeze", freeze)
|
||||
assert (
|
||||
assemble.main(
|
||||
[
|
||||
str(catalog_path),
|
||||
str(verification_path),
|
||||
"--repos",
|
||||
str(repos_path),
|
||||
"--used",
|
||||
str(used_path),
|
||||
"--out",
|
||||
str(tmp_path / "out"),
|
||||
"--history-cache",
|
||||
str(cache),
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert json.loads(capsys.readouterr().out) == {"checked": True}
|
||||
|
||||
|
||||
def test_insufficient_negatives_and_invalid_ratio(pair, tmp_path, monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user