marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

unseen: filter advisory enumeration server-side (ecosystem/date/cwes)

Browse Source 
enumerate_advisories paged the entire global GitHub reviewed-advisory
database (every ecosystem, every year, every CWE) before extraction, so
a 300-repo draft never converged -- a live run reached 21,962 advisories
in 4.5h with extraction never started.

Push the existing candidate() rejections into the query: page per
ecosystem, published>=2019-01-01, and cwes restricted to the seven
families. The cwes filter uses bare CWE numbers (89), which is what the
GitHub advisories API matches -- "CWE-89" silently returns zero. The
whole cwes filter is omitted if the taxonomy ever holds a non-exact
token, so a selectable advisory is never dropped. Journal version 3
(ecosystem:page keys) refuses incompatible older journals.

Live page-1 smoke (pip, npm): 100/100 rows carry a family CWE, none
published before 2019. Offline: 166 unseen tests pass, ruff clean.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-08 08:19:30 +02:00
co-authored by Claude Opus 4.8
parent d7bf0ba750
commit 90e2400e6f
4 changed files with 181 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/unseen/draft.py
+16 -14
View File
@@ -174,7 +174,7 @@ def prove_source(git, revision: str) -> dict:
def advisory_index(rows: list[dict], github) -> dict[str, set[str]]:
"""Include ALL enumerated fixes, even old, unmapped, multi-fix and other ecosystems.
"""Include ALL fixes in the enumerated population, even from rejected candidates.
Canonicalize link names as well, so an old advisory URL survives repository renames.
A 404 is not treated as proof that an advisory is unrelated to a selected repository.
@@ -219,7 +219,7 @@ def inputs(used, seed, known_empty, clones):
"used_set_digest": digest({label: sorted(names) for label, names in used.sources.items()}),
"known_empty": sorted(known_empty),
"clone_limit_bytes": getattr(clones, "limit_bytes", 500 * 1024 * 1024),
"journal_version": 2,
"journal_version": 3,
}
# Hardcoded design thresholds and algorithms are covered as well as taxonomy data.
for name in ("draft", "source", "extract", "eligibility", "journal"):
@@ -229,18 +229,20 @@ def inputs(used, seed, known_empty, clones):
def enumerate_advisories(github, journal):
page, records = 1, {}
while True:
key = str(page)
if key not in journal.rows["advisories"]:
journal.boundary()
result = github.advisory_page(page)
journal.put("advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "done": result["done"]})
result = journal.rows["advisories"][key]
records.update((row["ghsa_id"], row) for row in result["rows"])
if result["done"]:
break
page += 1
records = {}
for ecosystem in source.ECOSYSTEMS:
page = 1
while True:
key = f"{ecosystem}:{page}"
if key not in journal.rows["advisories"]:
journal.boundary()
result = github.advisory_page(page, ecosystem=ecosystem)
journal.put("advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "done": result["done"]})
result = journal.rows["advisories"][key]
records.update((row["ghsa_id"], row) for row in result["rows"])
if result["done"]:
break
page += 1
if not records:
raise InstrumentFailure("empty_advisories")
return list(records.values())
benchmarks/unseen/source.py
+34 -12
View File
@@ -269,24 +269,46 @@ class GitHub:
"text": base64.b64decode(row.get("content", "")).decode("utf-8", errors="replace"),
}
def advisory_page(self, page: int) -> dict:
def advisory_page(self, page: int, *, ecosystem: str) -> dict:
if ecosystem not in ECOSYSTEMS:
raise ValueError("invalid_ecosystem")
# Oldest first: newly published advisories do not shift already persisted pages.
rows = self.get(
"/advisories?" + urlencode({"type": "reviewed", "per_page": 100, "page": page, "sort": "published", "direction": "asc"})
)
parameters = {
"type": "reviewed",
"per_page": 100,
"page": page,
"sort": "published",
"direction": "asc",
"ecosystem": ecosystem,
# Inclusive syntax used in benchmarks/independent/research-2026-09-30-php.md.
"published": ">=2019-01-01",
}
taxonomy = load_families()
cwes = {cwe for family in taxonomy.families if family.id in FAMILIES for cwe in family.cwes}
# family_of_cwe uses exact membership. If the taxonomy gains non-exact tokens,
# fetch broadly rather than risk excluding a client-selectable advisory.
# GitHub's advisories `cwes` filter expects bare numbers (89), not "CWE-89".
if cwes and all(re.fullmatch(r"CWE-[1-9][0-9]*", cwe) for cwe in cwes):
parameters["cwes"] = ",".join(sorted((cwe.removeprefix("CWE-") for cwe in cwes), key=int))
rows = self.get("/advisories?" + urlencode(parameters))
if not isinstance(rows, list):
raise ValueError("invalid_advisory_page")
return {"rows": rows, "done": len(rows) < 100}
def advisories(self):
# Enumerate every reviewed advisory, including non-candidate fixes.
page = 1
while True:
result = self.advisory_page(page)
yield from result["rows"]
if result["done"]:
break
page += 1
"""Enumerate the candidate population, deduplicating multi-ecosystem advisories."""
seen = set()
for ecosystem in ECOSYSTEMS:
page = 1
while True:
result = self.advisory_page(page, ecosystem=ecosystem)
for row in result["rows"]:
if row["ghsa_id"] not in seen:
seen.add(row["ghsa_id"])
yield row
if result["done"]:
break
page += 1
class OSV:
tests/test_unseen_draft.py
+76 -12
View File
@@ -118,6 +118,7 @@ class FakeAPI:
def __init__(self, count=300):
self.count = count
self.licenses = []
self.advisory_pages = []
def get_repo(self, name):
self.calls += 1
@@ -138,9 +139,11 @@ class FakeAPI:
# P is MIT but actual introducing head is GPL: public output must use V.
return {"text": "", "spdx_id": "MIT" if ref == "b" * 40 else "GPL-3.0", "path": "LICENSE"}
def advisory_page(self, page):
def advisory_page(self, page, *, ecosystem):
self.calls += 1
rows = list(self.advisories())[(page - 1) * 100 : page * 100]
self.advisory_pages.append((ecosystem, page))
population = list(self.advisories()) if ecosystem == "pip" else []
rows = population[(page - 1) * 100 : page * 100]
return {"rows": rows, "done": len(rows) < 100}
def advisories(self):
@@ -457,8 +460,8 @@ def test_max_hours_stops_after_completed_page(tmp_path):
now = [0.0]
class TimedAPI(FakeAPI):
def advisory_page(self, page):
result = super().advisory_page(page)
def advisory_page(self, page, *, ecosystem):
result = super().advisory_page(page, ecosystem=ecosystem)
now[0] += 3601
return result
@@ -541,8 +544,8 @@ def test_advisory_journal_reduces_nested_blobs_and_resumes_by_page(tmp_path):
class Pages:
calls = []
def advisory_page(self, page):
self.calls.append(page)
def advisory_page(self, page, *, ecosystem):
self.calls.append((ecosystem, page))
return {"rows": [raw], "done": True, "unused": "raw page blob"}
api, used = Pages(), e.UsedSet()
@@ -551,14 +554,15 @@ def test_advisory_journal_reduces_nested_blobs_and_resumes_by_page(tmp_path):
with d.Journal(path).open(parameters) as journal:
rows = d.enumerate_advisories(api, journal)
assert rows == [expected]
saved = json.loads((path / "advisories.jsonl").read_bytes())
assert saved["value"] == {"rows": [expected], "done": True}
saved = [json.loads(line) for line in (path / "advisories.jsonl").read_bytes().splitlines()]
assert [row["id"] for row in saved] == [f"{eco}:1" for eco in s.ECOSYSTEMS]
assert all(row["value"] == {"rows": [expected], "done": True} for row in saved)
assert b"unused" not in (path / "advisories.jsonl").read_bytes()
assert b"raw advisory blob" not in (path / "advisories.jsonl").read_bytes()
assert b"raw page blob" not in (path / "advisories.jsonl").read_bytes()
with d.Journal(path).open(parameters) as journal:
resumed = d.enumerate_advisories(api, journal)
assert api.calls == [1] and resumed == rows
assert api.calls == [(eco, 1) for eco in s.ECOSYSTEMS] and resumed == rows
class Bracket(FakeOSV):
def get(self, identifier):
@@ -599,15 +603,75 @@ def test_reduced_advisories_preserve_all_fix_exclusions_and_selection():
s.candidate(s.reduce_advisory(raw), FakeAPI(), FakeOSV())
def test_old_raw_advisory_journal_version_refused_before_calls(tmp_path):
@pytest.mark.parametrize("old_version", [1, 2])
def test_old_advisory_journal_version_refused_before_calls(tmp_path, old_version):
used, api, osv, clones = e.UsedSet(), FakeAPI(1), FakeOSV(), FakeClones()
path = tmp_path / "journal"
parameters = d.inputs(used, 7, set(used.sources), clones)
assert parameters["journal_version"] == 2
with d.Journal(path).open({**parameters, "journal_version": 1}) as journal:
assert parameters["journal_version"] == 3
with d.Journal(path).open({**parameters, "journal_version": old_version}) as journal:
journal.put("advisories", "1", {"rows": list(api.advisories()), "done": True})
report = d.run(
tmp_path / "repo", github=api, osv=osv, clones=clones, used=used, seed=7, known_empty=set(used.sources), journal_path=path
)
assert report["status"] == "resume_refused" and report["changed_parameters"] == ["journal_version"]
assert api.calls == osv.calls == 0 and clones.counts == {}
def test_enumerate_ecosystem_pages_dedupes_and_resumes_partial_stream(tmp_path):
rows = list(FakeAPI(3).advisories())
pages = {
("pip", 1): {"rows": [rows[0]], "done": False},
("pip", 2): {"rows": [rows[1]], "done": True},
("npm", 1): {"rows": [rows[0], rows[2]], "done": True},
("maven", 1): {"rows": [], "done": True},
("composer", 1): {"rows": [], "done": True},
}
calls = []
class Pages:
def advisory_page(self, page, *, ecosystem):
calls.append((ecosystem, page))
return pages[ecosystem, page]
path = tmp_path / "journal"
with d.Journal(path).open({"journal_version": 3}) as journal:
journal.put("advisories", "pip:1", pages["pip", 1])
with d.Journal(path).open({"journal_version": 3}) as journal:
result = d.enumerate_advisories(Pages(), journal)
assert [row["ghsa_id"] for row in result] == [row["ghsa_id"] for row in rows]
assert list(journal.rows["advisories"]) == [f"{eco}:{page}" for eco, page in pages]
assert calls == list(pages)[1:]
with d.Journal(path).open({"journal_version": 3}) as journal:
assert d.enumerate_advisories(Pages(), journal) == result
assert calls == list(pages)[1:]
def test_filtered_http_population_reaches_draft_floor_with_bounded_pages(tmp_path, monkeypatch):
from urllib.parse import parse_qs, urlsplit
population = list(FakeAPI(300).advisories())
pages = []
class FilteredTransport:
def get(self, url, headers):
query = parse_qs(urlsplit(url).query)
# An unfiltered request would enumerate the global database: fail immediately.
assert query["ecosystem"][0] in s.ECOSYSTEMS
assert query["published"] == [">=2019-01-01"]
assert "89" in query["cwes"][0].split(",")
eco, page = query["ecosystem"][0], int(query["page"][0])
pages.append((eco, page))
assert len(pages) <= 7
rows = population if eco == "pip" else []
return rows[(page - 1) * 100 : page * 100]
api, used = FakeAPI(), e.UsedSet()
http = s.GitHub("fake", FilteredTransport())
monkeypatch.setattr(api, "advisory_page", http.advisory_page)
changes = entries()[0]["changes"]
monkeypatch.setattr(d, "repository_changes", lambda *a, **kw: (changes, {}, {"bytes": 12, "files": 1}))
report = d.run(tmp_path, github=api, osv=FakeOSV(), clones=FakeClones(), used=used, seed=7, known_empty=set(used.sources))
assert report["status"] == "draft" and report["repositories"] == 300
assert report["advisories"] == 300
assert pages == [("pip", n) for n in range(1, 5)] + [(eco, 1) for eco in s.ECOSYSTEMS[1:]]
tests/test_unseen_source.py
+55 -1
View File
@@ -122,7 +122,9 @@ def test_http_adapters_with_fake_transport():
parsed = urlsplit(url)
if parsed.path == "/advisories":
page = int(parse_qs(parsed.query)["page"][0])
return [advisory()] * (100 if page == 1 else 1)
return [
{**advisory(), "ghsa_id": f"GHSA-test-test-{i:04d}"} for i in range(0 if page == 1 else 100, 100 if page == 1 else 101)
]
if "/license" in parsed.path:
assert parse_qs(parsed.query)["ref"] == ["b" * 40]
return {
@@ -271,3 +273,55 @@ def test_transport_decode_retry_respects_response_retry_after():
transport = s.JSONTransport(opener=lambda *a, **kw: responses.pop(0), sleeper=waits.append)
assert transport.get("https://example.invalid", {}) == {}
assert waits == [9]
@pytest.mark.parametrize("ecosystem", s.ECOSYSTEMS)
def test_advisory_page_filters_match_candidate_population(ecosystem):
from urllib.parse import parse_qs, urlsplit
taxonomy = s.load_families()
expected = {c for f in taxonomy.families if f.id in s.FAMILIES for c in f.cwes}
class Transport:
def get(self, url, headers):
query = parse_qs(urlsplit(url).query)
assert query == {
"type": ["reviewed"],
"per_page": ["100"],
"page": ["2"],
"sort": ["published"],
"direction": ["asc"],
"ecosystem": [ecosystem],
"published": [">=2019-01-01"],
"cwes": [",".join(sorted((c.removeprefix("CWE-") for c in expected), key=int))],
}
return []
assert s.GitHub("fake", Transport()).advisory_page(2, ecosystem=ecosystem) == {"rows": [], "done": True}
# Every enumerated CWE is selectable at the inclusive date boundary.
for cwe in expected:
row = advisory()
row.update(published_at="2019-01-01T00:00:00Z", cwes=[{"cwe_id": cwe}])
row["vulnerabilities"][0]["package"]["ecosystem"] = ecosystem
assert s.candidate(row, API(), OSV()).family == taxonomy.family_of_cwe(cwe).id
@pytest.mark.parametrize("token", ["CWE-89*", "CWE-", "89", "CWE-79..CWE-89"])
def test_advisory_page_omits_entire_cwe_filter_for_non_exact_taxonomy(monkeypatch, token):
from dataclasses import replace
from urllib.parse import parse_qs, urlsplit
taxonomy = s.load_families()
family = taxonomy.by_id("injection")
taxonomy = replace(taxonomy, families=tuple(replace(f, cwes=f.cwes | {token}) if f == family else f for f in taxonomy.families))
monkeypatch.setattr(s, "load_families", lambda: taxonomy)
class Transport:
def get(self, url, headers):
query = parse_qs(urlsplit(url).query)
assert "cwes" not in query
assert query["published"] == [">=2019-01-01"]
assert query["ecosystem"] == ["pip"]
return []
s.GitHub("fake", Transport()).advisory_page(1, ecosystem="pip")