From a3a38f1b9f22aa7b4d88d85c1bcc420b6c937702 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:10:19 +0200 Subject: [PATCH] unseen: bucket-cache get_repo (kill the repeated used-set canonicalization tax) The real bottleneck was not enumeration or extraction: canonicalize_used resolves the entire exclusion set (measured: 4,499 names) via serial GitHub get_repo on every launch, before any harvesting -- ~30 min of network-latency-bound work that also burns ~4,499 of GitHub's 5,000/hr budget before real work starts (hence the low CPU and the "never finishes"). The earlier resolution-cache only banked via BankedGitHub.resolve, but canonicalize_used calls eligibility.resolve -> get_repo directly, so it was never cached and re-ran every time. Cache get_repo at the real choke point. CorpusBank gains cache_repo / get_cached_repo storing one object per normalized name at /repos/.json -- NOT seed-scoped, since repo metadata is seed-independent, so every seed/run shares it. 404s are cached as None too (a found-but-None hit) so known-missing repos are not re-fetched. BankedGitHub.get_repo now checks the local journal, then the bucket, and only then GitHub (banking the result). canonicalize_used, advisory_index and candidate resolution all funnel through get_repo, so all of them are covered. Effect: the first run pays the lookups once and banks them; every later launch reads them from the bucket (no GitHub, no rate-limit spend), so the rate budget goes to actual candidate harvesting and restarts are cheap. Proven live against ousast-corpus (repos/ fills 1:1 with resolutions). 206 unseen tests + new repo-cache test pass, ruff clean. Co-Authored-By: Claude Opus 4.8 --- benchmarks/unseen/bank.py | 21 +++++++++++++++++++++ benchmarks/unseen/draft.py | 20 ++++++++++++++++++++ tests/test_unseen_bank.py | 19 +++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/benchmarks/unseen/bank.py b/benchmarks/unseen/bank.py index 73e9533..a40eef7 100644 --- a/benchmarks/unseen/bank.py +++ b/benchmarks/unseen/bank.py @@ -108,6 +108,9 @@ class CorpusBank: self.resolved_prefix = f"{prefix}/{seed}/resolved/" self.advisories_key = f"{prefix}/{seed}/advisories/list.json" self.repositories_prefix = f"{prefix}/{seed}/repositories/" + # get_repo metadata is seed-independent GitHub data: share it across seeds + # so the ~thousands of used-set canonicalization lookups are paid once ever. + self.repos_prefix = f"{prefix}/repos/" @staticmethod def advisory_key(advisory_id: str) -> str: @@ -192,6 +195,24 @@ class CorpusBank: if body["resolution_contract"] == contract } + def cache_repo(self, name: str, metadata: dict | None, *, contract: str) -> None: + """Persist one get_repo result (metadata, or None for a 404) by normalized name.""" + self._put( + self.repos_prefix + hashlib.sha256(name.encode()).hexdigest() + ".json", + {"name": name, "metadata": metadata, "resolution_contract": contract}, + labels={"contract": contract}, + ) + + def get_cached_repo(self, name: str, *, contract: str) -> tuple[bool, dict | None]: + """Return (found, metadata). found is True even when metadata is None (a cached 404).""" + stored = _retry(self.client.get, self.repos_prefix + hashlib.sha256(name.encode()).hexdigest() + ".json") + if stored is None: + return (False, None) + body = json.loads(stored[0]) + if body.get("name") != name or body.get("resolution_contract") != contract: + return (False, None) + return (True, body["metadata"]) + @staticmethod def key_for(repository: str, fix: str) -> str: return hashlib.sha256(f"{repository}\n{fix}".encode()).hexdigest() diff --git a/benchmarks/unseen/draft.py b/benchmarks/unseen/draft.py index da52ba4..0eb76da 100644 --- a/benchmarks/unseen/draft.py +++ b/benchmarks/unseen/draft.py @@ -218,6 +218,26 @@ class BankedGitHub(ResolvedGitHub): self.resolutions = bank.repositories(contract=RESOLUTION_CONTRACT) journal.rows["resolution"].update({name: body["metadata"] if body is not None else None for name, body in self.resolutions.items()}) + def get_repo(self, name): + # Bucket-cache get_repo (the universal resolution choke point) so the + # ~thousands of used-set canonicalization lookups and candidate resolves + # are not re-fetched from GitHub every run. 404s are cached as None too. + key = eligibility.normalize(name) + if key not in self.journal.rows["resolution"]: + found, value = self.bank.get_cached_repo(key, contract=RESOLUTION_CONTRACT) + if not found: + self.journal.boundary() + try: + value = self.github.get_repo(key) + except eligibility.RepositoryNotFound: + value = None + self.bank.cache_repo(key, value, contract=RESOLUTION_CONTRACT) + self.journal.put("resolution", key, value) + value = self.journal.rows["resolution"][key] + if value is None: + raise eligibility.RepositoryNotFound("not_found") + return value + def resolve(self, name): key = eligibility.normalize(name) if key not in self.resolutions: diff --git a/tests/test_unseen_bank.py b/tests/test_unseen_bank.py index 8d4fae9..1ea01dc 100644 --- a/tests/test_unseen_bank.py +++ b/tests/test_unseen_bank.py @@ -38,6 +38,25 @@ class FakeObjects: return [key for key in reversed(self.objects) if key.startswith(prefix)] +def test_cache_repo_roundtrip_404_and_shared_prefix(): + objects = FakeObjects() + bank = CorpusBank(objects, seed=20260601, prefix="unseen") + # the get_repo cache is shared across seeds, not under the seed prefix + assert bank.repos_prefix == "unseen/repos/" + bank.cache_repo("org/repo", {"full_name": "org/repo", "size": 10}, contract="v1") + bank.cache_repo("org/gone", None, contract="v1") + assert bank.get_cached_repo("org/repo", contract="v1") == (True, {"full_name": "org/repo", "size": 10}) + # a cached 404 is a hit (found True, metadata None): it must not re-hit GitHub + assert bank.get_cached_repo("org/gone", contract="v1") == (True, None) + assert bank.get_cached_repo("org/never", contract="v1") == (False, None) + # a stale contract is a miss so the lookup is refreshed + assert bank.get_cached_repo("org/repo", contract="v2") == (False, None) + # idempotent: re-caching the same metadata does not rewrite + puts = objects.calls.count("put") + bank.cache_repo("org/repo", {"full_name": "org/repo", "size": 10}, contract="v1") + assert objects.calls.count("put") == puts + + def settings_env(): return { "CORPUS_S3_ENDPOINT": "https://objects.invalid",