From 00a1fdb30f86310fdcced21d15ab13e597ffb6e4 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:53:05 +0200 Subject: [PATCH] unseen: full-objects clone instead of blob:none promisor (bulk pack, no lazy per-blob fetch) The harvester cloned --filter=blob:none --no-checkout, then blame/log/diff triggered the promisor to fetch each needed blob ON DEMAND -- hundreds of round-trips per repo, the dominant cost that made the draw unable to converge. The blobless clone was a premature optimization that backfired. Drop --filter=blob:none from the clone and the two commit fetches. A full-objects --no-checkout clone brings the whole commit graph AND every blob in one pack, so blame/log/diff read locally; --no-checkout still skips the working tree so disk stays bounded (plus the existing 500 MiB clone_size_cap and the disk guard). With no promisor remote, a genuinely missing object now errors loudly instead of silently round-tripping -- the instrument guarantee that the bulk fetch was complete. Extraction semantics are unchanged (same SZZ introducing-commit positive, same stratified ordinary negatives, byte-identical entries); only the object-transfer strategy changes. Test inverted to assert no blob:none filter on the clone; README updated. 145 unseen tests pass, ruff clean. Co-Authored-By: Claude Opus 4.8 --- benchmarks/unseen/README.md | 13 ++++++++----- benchmarks/unseen/draft.py | 13 +++++++++---- tests/test_unseen_draft.py | 6 +++++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/benchmarks/unseen/README.md b/benchmarks/unseen/README.md index 5ca7d9d..94c04ae 100644 --- a/benchmarks/unseen/README.md +++ b/benchmarks/unseen/README.md @@ -101,11 +101,14 @@ passing implementation tests does not assert that 300 eligible repositories exis Clones and checkpoints default to `results_root()/unseen-draft` (normally `~/ousast-results/plane/unseen-draft`), overridable by `--cache /outside/path`. The cache cannot lie inside `--root`. One opaque, disposable clone exists at a time; -it is removed on success, rejection, or failure. `--filter=blob:none --no-checkout` -keeps the commit graph needed for temporal thirds, blame, and release-tag ancestry, -and fetches source blobs on demand. This is a **partial clone**, not a shallow history. -The 500 MiB disk cap is checked after Git commands, including lazy blob fetches; -one in-flight command can transiently exceed it before rejection and cleanup. +it is removed on success, rejection, or failure. A full-objects `--no-checkout` clone +brings the whole commit graph and every blob in one pack, so temporal thirds, blame, +and release-tag ancestry read locally; `--no-checkout` still skips the working tree. +This is a full clone, not a shallow history, and has no blob:none promisor, so a +genuinely missing object errors loudly instead of silently round-tripping (the old +promisor path fetched blobs one at a time, hundreds of round-trips per repo). The +500 MiB disk cap is checked after Git commands; one in-flight command can transiently +exceed it before rejection and cleanup. Git output is captured, never forwarded. Each repository must read a nonempty source blob before extraction. Summary instrumentation includes source read operations and actual bytes, clone count, cumulative/peak disk bytes, wall time, and logical API calls diff --git a/benchmarks/unseen/draft.py b/benchmarks/unseen/draft.py index 699aacc..0770126 100644 --- a/benchmarks/unseen/draft.py +++ b/benchmarks/unseen/draft.py @@ -166,9 +166,14 @@ class Clones: git = Git(path, runner=self.runner, limit_bytes=self.limit_bytes, deadline_seconds=self.deadline_seconds) try: # Full commit graph is needed for temporal thirds and OSV brackets. - # Content reads fetch blobs on demand through Git's promisor mechanism. + # A full-objects clone (no blob:none promisor) brings every blob in + # one pack, so blame/log/diff read locally instead of fetching each + # blob on demand (the old promisor path cost ~hundreds of round-trips + # per repo). --no-checkout still skips the working tree. With no + # promisor remote, a genuinely missing object errors loudly rather + # than silently round-tripping. Disk is bounded by limit_bytes below. done = self.runner( - ["git", "-c", "credential.helper=", "clone", "--quiet", "--filter=blob:none", "--no-checkout", url, str(path)], + ["git", "-c", "credential.helper=", "clone", "--quiet", "--no-checkout", url, str(path)], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -229,7 +234,7 @@ def repository_changes(git, candidate, fixes: set[str], *, seed: int): # Extra advisory fixes are best-effort exclusions. The candidate # was fetched strictly by extract_repository. Do not import extra # tags before introducing() checks the candidate's version bracket. - git.run("fetch", "--quiet", "--no-tags", "--filter=blob:none", "origin", fix) + git.run("fetch", "--quiet", "--no-tags", "origin", fix) resolved.append(git.run("rev-parse", fix).strip()) except InstrumentFailure as exc: if fix == candidate.fix or not str(exc).startswith("git_exit_"): @@ -325,7 +330,7 @@ def extract_repository(candidate, github, clones, fixes, seed): try: with extract.process_deadline(total): with clones.open(candidate.url) as git: - git.run("fetch", "--quiet", "--filter=blob:none", "origin", candidate.fix) + git.run("fetch", "--quiet", "origin", candidate.fix) changes, exclusions, proof = repository_changes(git, candidate, fixes, seed=seed) entry = {**asdict(candidate), "changes": changes, "instrument": proof} result = {"entry": entry, "ordinary_exclusions": exclusions} diff --git a/tests/test_unseen_draft.py b/tests/test_unseen_draft.py index 89912b4..f29d063 100644 --- a/tests/test_unseen_draft.py +++ b/tests/test_unseen_draft.py @@ -100,7 +100,11 @@ def test_cache_is_outside_repository_deleted_and_partial(tmp_path): assert git.path.exists() clone_path = git.path assert not clone_path.exists() - assert "--filter=blob:none" in calls[0] and "--no-checkout" in calls[0] + # Full-objects clone: no blob:none promisor (which caused per-blob lazy + # fetches), so every blob arrives in one pack and a missing object errors + # loudly. --no-checkout still skips the working tree. + assert "--filter=blob:none" not in calls[0] and "--no-checkout" in calls[0] + assert "clone" in calls[0] assert clones.counts["clones"] == 1 and clones.counts["clone_bytes"] == 3