marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-17 03:59:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

unseen: cache enumeration + resolution to ousast-corpus (restarts skip the ~26-min prefix)

Browse Source 
The decision bank made extraction durable, but every restart still
re-paid the ~26-minute advisory enumeration + per-advisory resolution,
since those lived only in the local journal. Mirror them into the bucket
so a restart reads them back and goes essentially straight to any
remaining extraction.

bank.py: bank_resolution/resolved at <prefix>/<seed>/resolved/<sha256(ghsa)>.json
(streamed one-by-one as each advisory is decided), cache_advisories/
cached_advisories at <prefix>/<seed>/advisories/list.json, and
bank_repository/repositories (caches the advisory_index repo-resolution
pass, preserving renames and 404s). All gated by RESOLUTION_CONTRACT="v1"
(extract.py) -- bumped only when enumeration/eligibility/source.candidate
output changes; recent fetch/parse/pool changes do not bump it.

draft.py run(): with --bank, advisories load from the cache (else
enumerate then cache); eligibility decisions load from the bucket
(contract-matched) to skip re-resolution; new resolutions stream to the
bucket before the local checkpoint write. Repository lookups reuse the
repositories/ cache. No --bank => local-journal behavior unchanged.

Net: advisories + resolutions + extraction all resume from ousast-corpus;
restarts no longer re-enumerate or re-resolve. Fresh and resumed runs
produce byte-identical manifests at 1 and 4 workers. Bank methods proven
live against ousast-corpus. 356 tests pass, ruff clean.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-10 16:33:29 +02:00
co-authored by Claude Opus 4.8
parent 857263d975
commit 06b8a40c46
5 changed files with 477 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/unseen/bank.py
+86
View File
@@ -105,6 +105,92 @@ class CorpusBank:
self.client = client
self.seed = seed
self.decisions_prefix = f"{prefix}/{seed}/decisions/"
self.resolved_prefix = f"{prefix}/{seed}/resolved/"
self.advisories_key = f"{prefix}/{seed}/advisories/list.json"
self.repositories_prefix = f"{prefix}/{seed}/repositories/"
@staticmethod
def advisory_key(advisory_id: str) -> str:
return hashlib.sha256(advisory_id.encode()).hexdigest()
def _put(self, key: str, body: dict, *, labels: dict) -> None:
existing = _retry(self.client.get, key)
if existing is not None:
previous = json.loads(existing[0])
if {field: value for field, value in previous.items() if field != "ts"} == body:
return
body = {**body, "ts": datetime.now(UTC).isoformat()}
data = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
_retry(self.client.put, key, data, labels=labels)
def bank_resolution(self, advisory_id: str, result: dict, *, contract: str) -> None:
decision = "candidate" if "candidate" in result else "reject"
self._put(
self.resolved_prefix + self.advisory_key(advisory_id) + ".json",
{
"advisory": advisory_id,
"decision": decision,
"result": result,
"reason": result.get("rejected"),
"resolution_contract": contract,
"seed": self.seed,
},
labels={"decision": decision, "contract": contract},
)
def _listed(self, prefix: str, identity: str) -> dict[str, dict]:
results = {}
for key in sorted(self.client.keys(prefix)):
stored = _retry(self.client.get, key)
if stored is None:
raise ValueError("corpus_resolution_missing")
body = json.loads(stored[0])
name = body[identity]
if key != prefix + self.advisory_key(name) + ".json" or body["seed"] != self.seed:
raise ValueError("corpus_resolution_identity")
results[name] = body
return results
def resolved(self) -> dict[str, dict]:
results = self._listed(self.resolved_prefix, "advisory")
for body in results.values():
expected = "candidate" if "candidate" in body["result"] else "reject"
if body["decision"] != expected:
raise ValueError("corpus_resolution_invalid")
return results
def cache_advisories(self, rows: list, *, contract: str) -> None:
if self.cached_advisories(contract=contract) is not None:
return
self._put(
self.advisories_key,
{"advisories": rows, "advisories_contract": contract, "seed": self.seed},
labels={"contract": contract},
)
def cached_advisories(self, *, contract: str) -> list | None:
stored = _retry(self.client.get, self.advisories_key)
if stored is None:
return None
body = json.loads(stored[0])
if body["seed"] != self.seed:
raise ValueError("corpus_advisories_identity")
return body["advisories"] if body["advisories_contract"] == contract else None
def bank_repository(self, name: str, result: dict | None, *, contract: str) -> None:
"""Retain redirects, fork identity and 404s needed by the all-fix index."""
self._put(
self.repositories_prefix + self.advisory_key(name) + ".json",
{"repository": name, "result": result, "resolution_contract": contract, "seed": self.seed},
labels={"contract": contract},
)
def repositories(self, *, contract: str) -> dict[str, dict | None]:
return {
name: body["result"]
for name, body in self._listed(self.repositories_prefix, "repository").items()
if body["resolution_contract"] == contract
}
@staticmethod
def key_for(repository: str, fix: str) -> str:
benchmarks/unseen/draft.py
+70 -14
View File
@@ -26,7 +26,7 @@ from pathlib import Path
from . import eligibility, extract, source
from .bank import CorpusBank, CorpusObjects, corpus_settings, split_decided
from .extract import EXTRACTION_CONTRACT, Git, InstrumentFailure
from .extract import EXTRACTION_CONTRACT, RESOLUTION_CONTRACT, Git, InstrumentFailure
from .journal import Journal, ResolvedGitHub, ResumeMismatch, Stopped, atomic, digest, external
@@ -209,6 +209,43 @@ def prove_source(git, revision: str) -> dict:
raise InstrumentFailure("zero_source_bytes")
class BankedGitHub(ResolvedGitHub):
"""Stream the fix index's repository metadata before continuing the draw."""
def __init__(self, github, journal, bank):
super().__init__(github, journal)
self.bank = bank
self.resolutions = bank.repositories(contract=RESOLUTION_CONTRACT)
journal.rows["resolution"].update({name: body["metadata"] if body is not None else None for name, body in self.resolutions.items()})
def resolve(self, name):
key = eligibility.normalize(name)
if key not in self.resolutions:
try:
resolved = eligibility.resolve(name, self)
except eligibility.RepositoryNotFound:
body = None
else:
body = {
"metadata": self.journal.rows["resolution"].get(key),
"full_name": resolved.full_name,
"names": sorted(resolved.names),
"network": resolved.network,
}
self.bank.bank_repository(key, body, contract=RESOLUTION_CONTRACT)
self.resolutions[key] = body
body = self.resolutions[key]
if body is None:
raise eligibility.RepositoryNotFound("not_found")
return eligibility.Resolution(body["full_name"], frozenset(body["names"]), body["network"])
def resolve_repository(name, github):
if isinstance(github, BankedGitHub):
return github.resolve(name)
return eligibility.resolve(name, github)
def advisory_index(rows: list[dict], github) -> dict[str, set[str]]:
"""Include ALL fixes in the enumerated population, even from rejected candidates.
@@ -219,7 +256,7 @@ def advisory_index(rows: list[dict], github) -> dict[str, set[str]]:
for row in rows:
for name, sha in source.fix_links(row):
try:
resolved = eligibility.resolve(name, github)
resolved = resolve_repository(name, github)
except eligibility.RepositoryNotFound:
index[name].add(sha)
continue
@@ -305,7 +342,7 @@ def decide(row, github, osv, used, selected_networks=()):
if len(links) != 1:
raise source.Rejected("fix_links")
name, _ = next(iter(links))
resolved = eligibility.resolve(name, github)
resolved = resolve_repository(name, github)
rejected_by = eligibility.rejection(resolved, used, set())
if rejected_by:
raise source.Rejected("used_" + rejected_by)
@@ -382,6 +419,7 @@ def ordered_extractions(
pool_size,
bank: CorpusBank | None = None,
decided=None,
stale_resolutions=(),
):
"""Bounded in-flight work, completion journaling, and seed-order acceptance.
@@ -400,6 +438,15 @@ def ordered_extractions(
context = multiprocessing.get_context("fork")
pool = ProcessPoolExecutor(max_workers=workers, mp_context=context)
def resolution(row):
key = row["ghsa_id"]
if key not in checkpoint.rows["eligibility"]:
result = decide(row, api, osv, used)
if bank is not None:
bank.bank_resolution(key, result, contract=RESOLUTION_CONTRACT)
checkpoint.put("eligibility", key, result)
return checkpoint.rows["eligibility"][key]
def remember(repository, fix, result):
bank.bank(repository, fix, result, contract=EXTRACTION_CONTRACT)
# Reuse decisions made during this invocation too: another advisory for
@@ -461,13 +508,15 @@ def ordered_extractions(
exhausted = True
break
key = row["ghsa_id"]
if key in stale_resolutions:
resolution(row)
identity = None
if bank is not None:
links = source.fix_links(row)
if len(links) == 1:
repository, fix = next(iter(links))
try:
resolved = eligibility.resolve(repository, api)
resolved = resolve_repository(repository, api)
repository = resolved.full_name
except eligibility.RepositoryNotFound:
resolved = None
@@ -508,9 +557,7 @@ def ordered_extractions(
checkpoint.put("extraction", extraction_key, result)
pending.append((row, decision, extraction_key))
break
if key not in checkpoint.rows["eligibility"]:
checkpoint.put("eligibility", key, decide(row, api, osv, used))
decision = checkpoint.rows["eligibility"][key]
decision = resolution(row)
extraction_key, future = None, None
if "candidate" in decision:
candidate = source.Candidate(**decision["candidate"])
@@ -619,6 +666,7 @@ def run(
scratch = ExitStack()
try:
decided = bank.decided() if bank is not None else {}
banked = bank.resolved() if bank is not None else {}
accepted, reusable = split_decided(decided, contract=EXTRACTION_CONTRACT)
decided = {key: value for key, value in decided.items() if key in reusable}
known_empty = set(known_empty or ())
@@ -638,16 +686,23 @@ def run(
)
with checkpoint.open(inputs(used, seed, known_empty, clones, pool_size=pool_size, slices=slices)):
checkpoint.boundary()
api = ResolvedGitHub(github, checkpoint)
api = BankedGitHub(github, checkpoint, bank) if bank is not None else ResolvedGitHub(github, checkpoint)
checkpoint.rows["eligibility"].update(
{key: body["result"] for key, body in banked.items() if body["resolution_contract"] == RESOLUTION_CONTRACT}
)
used = copy.deepcopy(used) # canonicalization must not mutate the resume input
eligibility.canonicalize_used(used, api)
report["used_set"] = used.report(known_empty)
try:
rows = enumerate_advisories(api, checkpoint)
except InstrumentFailure as exc:
if str(exc) != "empty_advisories" or not accepted:
raise
rows = []
rows = bank.cached_advisories(contract=RESOLUTION_CONTRACT) if bank is not None else None
if rows is None:
try:
rows = enumerate_advisories(api, checkpoint)
except InstrumentFailure as exc:
if str(exc) != "empty_advisories" or not accepted:
raise
rows = []
if bank is not None:
bank.cache_advisories(rows, contract=RESOLUTION_CONTRACT)
if bank is not None:
# Retain accepted entries even if their advisory disappeared upstream.
present = {row["ghsa_id"] for row in rows}
@@ -680,6 +735,7 @@ def run(
pool_size,
bank=bank,
decided=decided,
stale_resolutions={key for key, body in banked.items() if body["resolution_contract"] != RESOLUTION_CONTRACT},
) as outcomes:
for row, decision, result in outcomes:
counts[source.ecosystem(row)] += 1
benchmarks/unseen/extract.py
+2
View File
@@ -21,6 +21,8 @@ from .source import Candidate, Rejected
# Bump only when accept/reject outcomes or accepted entry bytes change.
EXTRACTION_CONTRACT = "v1"
# Bump only when enumeration, eligibility or source.candidate outputs change.
RESOLUTION_CONTRACT = "v1"
LANGUAGES = {".py", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".php", ".java"}
HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@", re.M)