mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 20:49:28 +00:00
unseen: cache enumeration + resolution to ousast-corpus (restarts skip the ~26-min prefix)
The decision bank made extraction durable, but every restart still re-paid the ~26-minute advisory enumeration + per-advisory resolution, since those lived only in the local journal. Mirror them into the bucket so a restart reads them back and goes essentially straight to any remaining extraction. bank.py: bank_resolution/resolved at <prefix>/<seed>/resolved/<sha256(ghsa)>.json (streamed one-by-one as each advisory is decided), cache_advisories/ cached_advisories at <prefix>/<seed>/advisories/list.json, and bank_repository/repositories (caches the advisory_index repo-resolution pass, preserving renames and 404s). All gated by RESOLUTION_CONTRACT="v1" (extract.py) -- bumped only when enumeration/eligibility/source.candidate output changes; recent fetch/parse/pool changes do not bump it. draft.py run(): with --bank, advisories load from the cache (else enumerate then cache); eligibility decisions load from the bucket (contract-matched) to skip re-resolution; new resolutions stream to the bucket before the local checkpoint write. Repository lookups reuse the repositories/ cache. No --bank => local-journal behavior unchanged. Net: advisories + resolutions + extraction all resume from ousast-corpus; restarts no longer re-enumerate or re-resolve. Fresh and resumed runs produce byte-identical manifests at 1 and 4 workers. Bank methods proven live against ousast-corpus. 356 tests pass, ruff clean. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
857263d975
commit
06b8a40c46
@@ -214,3 +214,104 @@ def test_import_does_not_require_boto3(monkeypatch):
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
assert module.CorpusBank.key_for("a", "b") == CorpusBank.key_for("a", "b")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"result",
|
||||
[
|
||||
{"candidate": {"repository": "org/é"}, "network": "org/root"},
|
||||
{"rejected": "used_pairs"},
|
||||
{"rejected": "family", "network": "org/root"},
|
||||
],
|
||||
)
|
||||
def test_resolution_roundtrip_idempotent_and_contract_replacement(result):
|
||||
objects = FakeObjects()
|
||||
bank = CorpusBank(objects, seed=42, prefix="custom")
|
||||
advisory = "GHSA-test/advisory"
|
||||
key = f"custom/42/resolved/{hashlib.sha256(advisory.encode()).hexdigest()}.json"
|
||||
assert bank.advisory_key(advisory) == hashlib.sha256(advisory.encode()).hexdigest()
|
||||
bank.bank_resolution(advisory, result, contract="old")
|
||||
first = objects.objects.copy()
|
||||
bank.bank_resolution(advisory, result, contract="old")
|
||||
assert objects.objects == first and objects.calls.count("put") == 1
|
||||
row = bank.resolved()[advisory]
|
||||
assert row == {
|
||||
"advisory": advisory,
|
||||
"result": result,
|
||||
"decision": "candidate" if "candidate" in result else "reject",
|
||||
"reason": result.get("rejected"),
|
||||
"resolution_contract": "old",
|
||||
"seed": 42,
|
||||
"ts": row["ts"],
|
||||
}
|
||||
assert list(objects.objects) == [key]
|
||||
assert datetime.fromisoformat(row["ts"]).utcoffset().total_seconds() == 0
|
||||
bank.bank_resolution(advisory, result, contract="v1")
|
||||
assert bank.resolved()[advisory]["resolution_contract"] == "v1"
|
||||
assert objects.calls.count("put") == 2
|
||||
|
||||
|
||||
def test_advisory_snapshot_written_once_per_contract():
|
||||
objects = FakeObjects()
|
||||
bank = CorpusBank(objects, seed=7)
|
||||
rows = [{"ghsa_id": "GHSA-é", "references": ["a", "b"]}]
|
||||
assert bank.cached_advisories(contract="v1") is None
|
||||
bank.cache_advisories(rows, contract="v1")
|
||||
original = objects.objects.copy()
|
||||
assert bank.cached_advisories(contract="v1") == rows
|
||||
assert bank.cached_advisories(contract="v2") is None
|
||||
bank.cache_advisories([], contract="v1")
|
||||
assert objects.objects == original and objects.calls.count("put") == 1
|
||||
body = json.loads(objects.objects["unseen/7/advisories/list.json"])
|
||||
assert body == {"advisories": rows, "advisories_contract": "v1", "seed": 7, "ts": body["ts"]}
|
||||
bank.cache_advisories([], contract="v2")
|
||||
assert bank.cached_advisories(contract="v2") == []
|
||||
assert bank.cached_advisories(contract="v1") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["resolution", "advisories", "repository"])
|
||||
@pytest.mark.parametrize("operation", ["get", "put"])
|
||||
@pytest.mark.parametrize("failures", [1, 2])
|
||||
def test_new_bank_writes_retry_once(method, operation, failures):
|
||||
objects = FakeObjects()
|
||||
bank = CorpusBank(objects, seed=7)
|
||||
objects.failures[operation] = failures
|
||||
|
||||
def write():
|
||||
if method == "resolution":
|
||||
bank.bank_resolution("GHSA-a", {"rejected": "family"}, contract="v1")
|
||||
elif method == "advisories":
|
||||
bank.cache_advisories([], contract="v1")
|
||||
else:
|
||||
bank.bank_repository("org/repo", None, contract="v1")
|
||||
|
||||
if failures == 2:
|
||||
with pytest.raises(OSError):
|
||||
write()
|
||||
assert objects.calls.count(operation) == 2
|
||||
else:
|
||||
write()
|
||||
assert len(objects.objects) == 1
|
||||
assert objects.failures[operation] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["resolved", "cached_advisories", "repositories"])
|
||||
@pytest.mark.parametrize("failures", [1, 2])
|
||||
def test_new_bank_reads_retry_once(method, failures):
|
||||
objects = FakeObjects()
|
||||
bank = CorpusBank(objects, seed=7)
|
||||
bank.bank_resolution("GHSA-a", {"rejected": "family"}, contract="v1")
|
||||
bank.cache_advisories([], contract="v1")
|
||||
bank.bank_repository("org/old", {"full_name": "org/new"}, contract="v1")
|
||||
bank.bank_repository("org/missing", None, contract="old")
|
||||
objects.failures["get"] = failures
|
||||
read = getattr(bank, method)
|
||||
kwargs = {} if method == "resolved" else {"contract": "v1"}
|
||||
if failures == 2:
|
||||
with pytest.raises(OSError):
|
||||
read(**kwargs)
|
||||
else:
|
||||
result = read(**kwargs)
|
||||
assert result is not None
|
||||
if method == "repositories":
|
||||
assert result == {"org/old": {"full_name": "org/new"}}
|
||||
|
||||
Reference in New Issue
Block a user