mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
unseen: cursor pagination for advisory enumeration
The GitHub /advisories endpoint ignores the `page` query parameter: it paginates with the Link header's rel="next" cursor. The page-number pager re-fetched the same first 100 rows forever and never terminated, so enumeration never completed and no draft converged (a run fetched 5137 "pip pages" holding only 100 unique advisories). Surface the successful response headers from JSONTransport, and walk the rel="next" cursor per ecosystem until it is absent. The ecosystem / published>=2019 / bare-number cwes filters are preserved on the first request. Journal keys stay ecosystem:n (now cursor-walk positions) with the saved next URL for resume; journal_version 4 refuses older journals. Live smoke via the production pager: 300 unique advisories pulled in 3 API calls, advancing past the first page. Offline: 170 unseen tests pass, ruff clean. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
2d8f4f9e22
commit
5620165540
@@ -219,7 +219,7 @@ def inputs(used, seed, known_empty, clones):
|
||||
"used_set_digest": digest({label: sorted(names) for label, names in used.sources.items()}),
|
||||
"known_empty": sorted(known_empty),
|
||||
"clone_limit_bytes": getattr(clones, "limit_bytes", 500 * 1024 * 1024),
|
||||
"journal_version": 3,
|
||||
"journal_version": 4,
|
||||
}
|
||||
# Hardcoded design thresholds and algorithms are covered as well as taxonomy data.
|
||||
for name in ("draft", "source", "extract", "eligibility", "journal"):
|
||||
@@ -232,15 +232,19 @@ def enumerate_advisories(github, journal):
|
||||
records = {}
|
||||
for ecosystem in source.ECOSYSTEMS:
|
||||
page = 1
|
||||
next_url = None
|
||||
while True:
|
||||
key = f"{ecosystem}:{page}"
|
||||
if key not in journal.rows["advisories"]:
|
||||
journal.boundary()
|
||||
result = github.advisory_page(page, ecosystem=ecosystem)
|
||||
journal.put("advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "done": result["done"]})
|
||||
result = github.advisory_page(next_url, ecosystem=ecosystem)
|
||||
journal.put(
|
||||
"advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "next_url": result["next_url"]}
|
||||
)
|
||||
result = journal.rows["advisories"][key]
|
||||
records.update((row["ghsa_id"], row) for row in result["rows"])
|
||||
if result["done"]:
|
||||
next_url = result["next_url"]
|
||||
if next_url is None:
|
||||
break
|
||||
page += 1
|
||||
if not records:
|
||||
|
||||
Reference in New Issue
Block a user