unseen: bucket-cache get_repo (kill the repeated used-set canonicalization tax)

The real bottleneck was not enumeration or extraction: canonicalize_used
resolves the entire exclusion set (measured: 4,499 names) via serial
GitHub get_repo on every launch, before any harvesting -- ~30 min of
network-latency-bound work that also burns ~4,499 of GitHub's 5,000/hr
budget before real work starts (hence the low CPU and the "never
finishes"). The earlier resolution-cache only banked via
BankedGitHub.resolve, but canonicalize_used calls eligibility.resolve ->
get_repo directly, so it was never cached and re-ran every time.

Cache get_repo at the real choke point. CorpusBank gains cache_repo /
get_cached_repo storing one object per normalized name at
<prefix>/repos/<sha256(name)>.json -- NOT seed-scoped, since repo
metadata is seed-independent, so every seed/run shares it. 404s are
cached as None too (a found-but-None hit) so known-missing repos are not
re-fetched. BankedGitHub.get_repo now checks the local journal, then the
bucket, and only then GitHub (banking the result). canonicalize_used,
advisory_index and candidate resolution all funnel through get_repo, so
all of them are covered.

Effect: the first run pays the lookups once and banks them; every later
launch reads them from the bucket (no GitHub, no rate-limit spend), so
the rate budget goes to actual candidate harvesting and restarts are
cheap. Proven live against ousast-corpus (repos/ fills 1:1 with
resolutions). 206 unseen tests + new repo-cache test pass, ruff clean.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

benchmarks/unseen/bank.py

@@ -108,6 +108,9 @@ class CorpusBank:
         self.resolved_prefix = f"{prefix}/{seed}/resolved/"
         self.advisories_key = f"{prefix}/{seed}/advisories/list.json"
         self.repositories_prefix = f"{prefix}/{seed}/repositories/"
+        # get_repo metadata is seed-independent GitHub data: share it across seeds
+        # so the ~thousands of used-set canonicalization lookups are paid once ever.
+        self.repos_prefix = f"{prefix}/repos/"
 
     @staticmethod
     def advisory_key(advisory_id: str) -> str:
@@ -192,6 +195,24 @@ class CorpusBank:
             if body["resolution_contract"] == contract
         }
 
+    def cache_repo(self, name: str, metadata: dict | None, *, contract: str) -> None:
+        """Persist one get_repo result (metadata, or None for a 404) by normalized name."""
+        self._put(
+            self.repos_prefix + hashlib.sha256(name.encode()).hexdigest() + ".json",
+            {"name": name, "metadata": metadata, "resolution_contract": contract},
+            labels={"contract": contract},
+        )
+
+    def get_cached_repo(self, name: str, *, contract: str) -> tuple[bool, dict | None]:
+        """Return (found, metadata). found is True even when metadata is None (a cached 404)."""
+        stored = _retry(self.client.get, self.repos_prefix + hashlib.sha256(name.encode()).hexdigest() + ".json")
+        if stored is None:
+            return (False, None)
+        body = json.loads(stored[0])
+        if body.get("name") != name or body.get("resolution_contract") != contract:
+            return (False, None)
+        return (True, body["metadata"])
+
     @staticmethod
     def key_for(repository: str, fix: str) -> str:
         return hashlib.sha256(f"{repository}\n{fix}".encode()).hexdigest()

benchmarks/unseen/draft.py

@@ -218,6 +218,26 @@ class BankedGitHub(ResolvedGitHub):
         self.resolutions = bank.repositories(contract=RESOLUTION_CONTRACT)
         journal.rows["resolution"].update({name: body["metadata"] if body is not None else None for name, body in self.resolutions.items()})
 
+    def get_repo(self, name):
+        # Bucket-cache get_repo (the universal resolution choke point) so the
+        # ~thousands of used-set canonicalization lookups and candidate resolves
+        # are not re-fetched from GitHub every run. 404s are cached as None too.
+        key = eligibility.normalize(name)
+        if key not in self.journal.rows["resolution"]:
+            found, value = self.bank.get_cached_repo(key, contract=RESOLUTION_CONTRACT)
+            if not found:
+                self.journal.boundary()
+                try:
+                    value = self.github.get_repo(key)
+                except eligibility.RepositoryNotFound:
+                    value = None
+                self.bank.cache_repo(key, value, contract=RESOLUTION_CONTRACT)
+            self.journal.put("resolution", key, value)
+        value = self.journal.rows["resolution"][key]
+        if value is None:
+            raise eligibility.RepositoryNotFound("not_found")
+        return value
+
     def resolve(self, name):
         key = eligibility.normalize(name)
         if key not in self.resolutions:

tests/test_unseen_bank.py

@@ -38,6 +38,25 @@ class FakeObjects:
         return [key for key in reversed(self.objects) if key.startswith(prefix)]
 
 
+def test_cache_repo_roundtrip_404_and_shared_prefix():
+    objects = FakeObjects()
+    bank = CorpusBank(objects, seed=20260601, prefix="unseen")
+    # the get_repo cache is shared across seeds, not under the seed prefix
+    assert bank.repos_prefix == "unseen/repos/"
+    bank.cache_repo("org/repo", {"full_name": "org/repo", "size": 10}, contract="v1")
+    bank.cache_repo("org/gone", None, contract="v1")
+    assert bank.get_cached_repo("org/repo", contract="v1") == (True, {"full_name": "org/repo", "size": 10})
+    # a cached 404 is a hit (found True, metadata None): it must not re-hit GitHub
+    assert bank.get_cached_repo("org/gone", contract="v1") == (True, None)
+    assert bank.get_cached_repo("org/never", contract="v1") == (False, None)
+    # a stale contract is a miss so the lookup is refreshed
+    assert bank.get_cached_repo("org/repo", contract="v2") == (False, None)
+    # idempotent: re-caching the same metadata does not rewrite
+    puts = objects.calls.count("put")
+    bank.cache_repo("org/repo", {"full_name": "org/repo", "size": 10}, contract="v1")
+    assert objects.calls.count("put") == puts
+
+
 def settings_env():
     return {
         "CORPUS_S3_ENDPOINT": "https://objects.invalid",