unseen: cache enumeration + resolution to ousast-corpus (restarts skip the ~26-min prefix)

The decision bank made extraction durable, but every restart still
re-paid the ~26-minute advisory enumeration + per-advisory resolution,
since those lived only in the local journal. Mirror them into the bucket
so a restart reads them back and goes essentially straight to any
remaining extraction.

bank.py: bank_resolution/resolved at <prefix>/<seed>/resolved/<sha256(ghsa)>.json
(streamed one-by-one as each advisory is decided), cache_advisories/
cached_advisories at <prefix>/<seed>/advisories/list.json, and
bank_repository/repositories (caches the advisory_index repo-resolution
pass, preserving renames and 404s). All gated by RESOLUTION_CONTRACT="v1"
(extract.py) -- bumped only when enumeration/eligibility/source.candidate
output changes; recent fetch/parse/pool changes do not bump it.

draft.py run(): with --bank, advisories load from the cache (else
enumerate then cache); eligibility decisions load from the bucket
(contract-matched) to skip re-resolution; new resolutions stream to the
bucket before the local checkpoint write. Repository lookups reuse the
repositories/ cache. No --bank => local-journal behavior unchanged.

Net: advisories + resolutions + extraction all resume from ousast-corpus;
restarts no longer re-enumerate or re-resolve. Fresh and resumed runs
produce byte-identical manifests at 1 and 4 workers. Bank methods proven
live against ousast-corpus. 356 tests pass, ruff clean.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-10 16:33:29 +02:00
co-authored by Claude Opus 4.8
parent 857263d975
commit 06b8a40c46
5 changed files with 477 additions and 15 deletions
+86
View File
@@ -105,6 +105,92 @@ class CorpusBank:
self.client = client
self.seed = seed
self.decisions_prefix = f"{prefix}/{seed}/decisions/"
self.resolved_prefix = f"{prefix}/{seed}/resolved/"
self.advisories_key = f"{prefix}/{seed}/advisories/list.json"
self.repositories_prefix = f"{prefix}/{seed}/repositories/"
@staticmethod
def advisory_key(advisory_id: str) -> str:
return hashlib.sha256(advisory_id.encode()).hexdigest()
def _put(self, key: str, body: dict, *, labels: dict) -> None:
existing = _retry(self.client.get, key)
if existing is not None:
previous = json.loads(existing[0])
if {field: value for field, value in previous.items() if field != "ts"} == body:
return
body = {**body, "ts": datetime.now(UTC).isoformat()}
data = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
_retry(self.client.put, key, data, labels=labels)
def bank_resolution(self, advisory_id: str, result: dict, *, contract: str) -> None:
decision = "candidate" if "candidate" in result else "reject"
self._put(
self.resolved_prefix + self.advisory_key(advisory_id) + ".json",
{
"advisory": advisory_id,
"decision": decision,
"result": result,
"reason": result.get("rejected"),
"resolution_contract": contract,
"seed": self.seed,
},
labels={"decision": decision, "contract": contract},
)
def _listed(self, prefix: str, identity: str) -> dict[str, dict]:
results = {}
for key in sorted(self.client.keys(prefix)):
stored = _retry(self.client.get, key)
if stored is None:
raise ValueError("corpus_resolution_missing")
body = json.loads(stored[0])
name = body[identity]
if key != prefix + self.advisory_key(name) + ".json" or body["seed"] != self.seed:
raise ValueError("corpus_resolution_identity")
results[name] = body
return results
def resolved(self) -> dict[str, dict]:
results = self._listed(self.resolved_prefix, "advisory")
for body in results.values():
expected = "candidate" if "candidate" in body["result"] else "reject"
if body["decision"] != expected:
raise ValueError("corpus_resolution_invalid")
return results
def cache_advisories(self, rows: list, *, contract: str) -> None:
if self.cached_advisories(contract=contract) is not None:
return
self._put(
self.advisories_key,
{"advisories": rows, "advisories_contract": contract, "seed": self.seed},
labels={"contract": contract},
)
def cached_advisories(self, *, contract: str) -> list | None:
stored = _retry(self.client.get, self.advisories_key)
if stored is None:
return None
body = json.loads(stored[0])
if body["seed"] != self.seed:
raise ValueError("corpus_advisories_identity")
return body["advisories"] if body["advisories_contract"] == contract else None
def bank_repository(self, name: str, result: dict | None, *, contract: str) -> None:
"""Retain redirects, fork identity and 404s needed by the all-fix index."""
self._put(
self.repositories_prefix + self.advisory_key(name) + ".json",
{"repository": name, "result": result, "resolution_contract": contract, "seed": self.seed},
labels={"contract": contract},
)
def repositories(self, *, contract: str) -> dict[str, dict | None]:
return {
name: body["result"]
for name, body in self._listed(self.repositories_prefix, "repository").items()
if body["resolution_contract"] == contract
}
@staticmethod
def key_for(repository: str, fix: str) -> str: