unseen: revert the batched-blob materialize (it failed git fetch on ~half of repos)

Git.materialize enumerated every object reachable from HEAD and tried to
`git fetch --no-filter` them by SHA. Git rejects that for real repos
("did not send all necessary objects" / "bad revision"), exiting 1 ->
InstrumentFailure(git_exit_1) that failed the whole repository. Measured
in a live draw: 326/670 attempts, ~49% of all failures, which froze the
accepted pool at 44/300. Reproduced on pallets/flask (26221 reachable,
9287 missing, fetch exit 1).

Git cannot batch-fetch arbitrary tree/blob objects by SHA, so remove
materialize and restore the original lazy path: the blob:none clone keeps
the commit graph and the introducing/blame/ordinary-history reads fetch
their blobs on demand through Git's promisor. The per-fix tolerant fetch
loop, the process-pool parallelism, the 25-with-zero-entries guard, the
per-repo deadline and resume are unchanged. Extraction content is
byte-identical (only the prefetch is gone).

217 unseen tests pass, ruff clean.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-10 05:06:24 +02:00
co-authored by Claude Opus 4.8
parent 40bfdb8e18
commit 0023f61ff8
4 changed files with 22 additions and 98 deletions
+1 -3
View File
@@ -154,7 +154,7 @@ class Clones:
git = Git(path, runner=self.runner, limit_bytes=self.limit_bytes, deadline_seconds=self.deadline_seconds)
try:
# Full commit graph is needed for temporal thirds and OSV brackets.
# repository_changes batches blobs before reading their content.
# Content reads fetch blobs on demand through Git's promisor mechanism.
done = self.runner(
["git", "-c", "credential.helper=", "clone", "--quiet", "--filter=blob:none", "--no-checkout", url, str(path)],
stdin=subprocess.DEVNULL,
@@ -222,8 +222,6 @@ def repository_changes(git, candidate, fixes: set[str], *, seed: int):
except InstrumentFailure as exc:
if fix == candidate.fix or not str(exc).startswith("git_exit_"):
raise
# Batch only resolved histories before diff/show/blame can fetch per-file blobs.
git.materialize(["HEAD", *resolved])
proof = prove_source(git, candidate.parent)
change = extract.introducing(git, candidate)
known = set()
+2 -34
View File
@@ -53,12 +53,12 @@ class Git:
raise ExtractionTimeout("extraction_timeout")
return min(180, remaining)
def run(self, *args: str, allowed=(0,), input_data: bytes | None = None) -> str:
def run(self, *args: str, allowed=(0,)) -> str:
timeout = self.command_timeout()
try:
done = self.runner(
["git", "-c", "core.quotePath=false", "-C", str(self.path), *args],
**({"stdin": subprocess.DEVNULL} if input_data is None else {"input": input_data}),
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=timeout,
@@ -75,38 +75,6 @@ class Git:
self.last_blob_bytes = len(done.stdout)
return done.stdout.decode("utf-8", errors="replace")
def materialize(self, revisions: list[str]) -> None:
"""Fetch missing historical blobs in one pack, without changing content reads.
ordinary_history inspects the complete first-parent history, including both
sides, attributes and non-source diffs. Blame -M/-C and log -L can walk all
ancestors and copy sources. Include the reachable trees, not only tip files,
so these commands keep exactly their original semantics. This is a safe
superset (merge-side trees included), still subject to the disk/deadline caps.
Missing-object enumeration itself does not trigger lazy blob fetches.
"""
data = ("\n".join(revisions) + "\n").encode()
objects = self.run("rev-list", "--objects", "--missing=print", "--stdin", input_data=data)
missing = sorted({line[1:] for line in objects.splitlines() if line.startswith("?")})
if not all(SHA.fullmatch(oid) for oid in missing):
raise InstrumentFailure("invalid_missing_object")
if missing:
# stdin avoids argv limits; explicit object wants avoid one promisor
# request per show/blame. --no-filter overrides clone's stored filter.
self.run(
"fetch",
"--quiet",
"--no-tags",
"--no-write-fetch-head",
"--no-filter",
"--stdin",
"origin",
input_data=("\n".join(missing) + "\n").encode(),
)
remaining = self.run("rev-list", "--objects", "--missing=print", "--stdin", input_data=data)
if any(line.startswith("?") for line in remaining.splitlines()):
raise InstrumentFailure("missing_blobs_after_fetch")
def disk_bytes(self) -> int:
# git churns .git/objects/pack (incl. transient *.rev) during on-demand blob
# fetches, so a path rglob just listed can vanish before stat(): skip it.