From 90e2400e6f67abae32884180f32bce02804daa75 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:19:30 +0200 Subject: [PATCH] unseen: filter advisory enumeration server-side (ecosystem/date/cwes) enumerate_advisories paged the entire global GitHub reviewed-advisory database (every ecosystem, every year, every CWE) before extraction, so a 300-repo draft never converged -- a live run reached 21,962 advisories in 4.5h with extraction never started. Push the existing candidate() rejections into the query: page per ecosystem, published>=2019-01-01, and cwes restricted to the seven families. The cwes filter uses bare CWE numbers (89), which is what the GitHub advisories API matches -- "CWE-89" silently returns zero. The whole cwes filter is omitted if the taxonomy ever holds a non-exact token, so a selectable advisory is never dropped. Journal version 3 (ecosystem:page keys) refuses incompatible older journals. Live page-1 smoke (pip, npm): 100/100 rows carry a family CWE, none published before 2019. Offline: 166 unseen tests pass, ruff clean. Co-Authored-By: Claude Opus 4.8 --- benchmarks/unseen/draft.py | 30 +++++++------ benchmarks/unseen/source.py | 46 ++++++++++++++----- tests/test_unseen_draft.py | 88 ++++++++++++++++++++++++++++++++----- tests/test_unseen_source.py | 56 ++++++++++++++++++++++- 4 files changed, 181 insertions(+), 39 deletions(-) diff --git a/benchmarks/unseen/draft.py b/benchmarks/unseen/draft.py index 2376fcd..cc8eaf4 100644 --- a/benchmarks/unseen/draft.py +++ b/benchmarks/unseen/draft.py @@ -174,7 +174,7 @@ def prove_source(git, revision: str) -> dict: def advisory_index(rows: list[dict], github) -> dict[str, set[str]]: - """Include ALL enumerated fixes, even old, unmapped, multi-fix and other ecosystems. + """Include ALL fixes in the enumerated population, even from rejected candidates. Canonicalize link names as well, so an old advisory URL survives repository renames. A 404 is not treated as proof that an advisory is unrelated to a selected repository. @@ -219,7 +219,7 @@ def inputs(used, seed, known_empty, clones): "used_set_digest": digest({label: sorted(names) for label, names in used.sources.items()}), "known_empty": sorted(known_empty), "clone_limit_bytes": getattr(clones, "limit_bytes", 500 * 1024 * 1024), - "journal_version": 2, + "journal_version": 3, } # Hardcoded design thresholds and algorithms are covered as well as taxonomy data. for name in ("draft", "source", "extract", "eligibility", "journal"): @@ -229,18 +229,20 @@ def inputs(used, seed, known_empty, clones): def enumerate_advisories(github, journal): - page, records = 1, {} - while True: - key = str(page) - if key not in journal.rows["advisories"]: - journal.boundary() - result = github.advisory_page(page) - journal.put("advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "done": result["done"]}) - result = journal.rows["advisories"][key] - records.update((row["ghsa_id"], row) for row in result["rows"]) - if result["done"]: - break - page += 1 + records = {} + for ecosystem in source.ECOSYSTEMS: + page = 1 + while True: + key = f"{ecosystem}:{page}" + if key not in journal.rows["advisories"]: + journal.boundary() + result = github.advisory_page(page, ecosystem=ecosystem) + journal.put("advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "done": result["done"]}) + result = journal.rows["advisories"][key] + records.update((row["ghsa_id"], row) for row in result["rows"]) + if result["done"]: + break + page += 1 if not records: raise InstrumentFailure("empty_advisories") return list(records.values()) diff --git a/benchmarks/unseen/source.py b/benchmarks/unseen/source.py index 60c5c5b..9c700a6 100644 --- a/benchmarks/unseen/source.py +++ b/benchmarks/unseen/source.py @@ -269,24 +269,46 @@ class GitHub: "text": base64.b64decode(row.get("content", "")).decode("utf-8", errors="replace"), } - def advisory_page(self, page: int) -> dict: + def advisory_page(self, page: int, *, ecosystem: str) -> dict: + if ecosystem not in ECOSYSTEMS: + raise ValueError("invalid_ecosystem") # Oldest first: newly published advisories do not shift already persisted pages. - rows = self.get( - "/advisories?" + urlencode({"type": "reviewed", "per_page": 100, "page": page, "sort": "published", "direction": "asc"}) - ) + parameters = { + "type": "reviewed", + "per_page": 100, + "page": page, + "sort": "published", + "direction": "asc", + "ecosystem": ecosystem, + # Inclusive syntax used in benchmarks/independent/research-2026-09-30-php.md. + "published": ">=2019-01-01", + } + taxonomy = load_families() + cwes = {cwe for family in taxonomy.families if family.id in FAMILIES for cwe in family.cwes} + # family_of_cwe uses exact membership. If the taxonomy gains non-exact tokens, + # fetch broadly rather than risk excluding a client-selectable advisory. + # GitHub's advisories `cwes` filter expects bare numbers (89), not "CWE-89". + if cwes and all(re.fullmatch(r"CWE-[1-9][0-9]*", cwe) for cwe in cwes): + parameters["cwes"] = ",".join(sorted((cwe.removeprefix("CWE-") for cwe in cwes), key=int)) + rows = self.get("/advisories?" + urlencode(parameters)) if not isinstance(rows, list): raise ValueError("invalid_advisory_page") return {"rows": rows, "done": len(rows) < 100} def advisories(self): - # Enumerate every reviewed advisory, including non-candidate fixes. - page = 1 - while True: - result = self.advisory_page(page) - yield from result["rows"] - if result["done"]: - break - page += 1 + """Enumerate the candidate population, deduplicating multi-ecosystem advisories.""" + seen = set() + for ecosystem in ECOSYSTEMS: + page = 1 + while True: + result = self.advisory_page(page, ecosystem=ecosystem) + for row in result["rows"]: + if row["ghsa_id"] not in seen: + seen.add(row["ghsa_id"]) + yield row + if result["done"]: + break + page += 1 class OSV: diff --git a/tests/test_unseen_draft.py b/tests/test_unseen_draft.py index bdadd31..835b56e 100644 --- a/tests/test_unseen_draft.py +++ b/tests/test_unseen_draft.py @@ -118,6 +118,7 @@ class FakeAPI: def __init__(self, count=300): self.count = count self.licenses = [] + self.advisory_pages = [] def get_repo(self, name): self.calls += 1 @@ -138,9 +139,11 @@ class FakeAPI: # P is MIT but actual introducing head is GPL: public output must use V. return {"text": "", "spdx_id": "MIT" if ref == "b" * 40 else "GPL-3.0", "path": "LICENSE"} - def advisory_page(self, page): + def advisory_page(self, page, *, ecosystem): self.calls += 1 - rows = list(self.advisories())[(page - 1) * 100 : page * 100] + self.advisory_pages.append((ecosystem, page)) + population = list(self.advisories()) if ecosystem == "pip" else [] + rows = population[(page - 1) * 100 : page * 100] return {"rows": rows, "done": len(rows) < 100} def advisories(self): @@ -457,8 +460,8 @@ def test_max_hours_stops_after_completed_page(tmp_path): now = [0.0] class TimedAPI(FakeAPI): - def advisory_page(self, page): - result = super().advisory_page(page) + def advisory_page(self, page, *, ecosystem): + result = super().advisory_page(page, ecosystem=ecosystem) now[0] += 3601 return result @@ -541,8 +544,8 @@ def test_advisory_journal_reduces_nested_blobs_and_resumes_by_page(tmp_path): class Pages: calls = [] - def advisory_page(self, page): - self.calls.append(page) + def advisory_page(self, page, *, ecosystem): + self.calls.append((ecosystem, page)) return {"rows": [raw], "done": True, "unused": "raw page blob"} api, used = Pages(), e.UsedSet() @@ -551,14 +554,15 @@ def test_advisory_journal_reduces_nested_blobs_and_resumes_by_page(tmp_path): with d.Journal(path).open(parameters) as journal: rows = d.enumerate_advisories(api, journal) assert rows == [expected] - saved = json.loads((path / "advisories.jsonl").read_bytes()) - assert saved["value"] == {"rows": [expected], "done": True} + saved = [json.loads(line) for line in (path / "advisories.jsonl").read_bytes().splitlines()] + assert [row["id"] for row in saved] == [f"{eco}:1" for eco in s.ECOSYSTEMS] + assert all(row["value"] == {"rows": [expected], "done": True} for row in saved) assert b"unused" not in (path / "advisories.jsonl").read_bytes() assert b"raw advisory blob" not in (path / "advisories.jsonl").read_bytes() assert b"raw page blob" not in (path / "advisories.jsonl").read_bytes() with d.Journal(path).open(parameters) as journal: resumed = d.enumerate_advisories(api, journal) - assert api.calls == [1] and resumed == rows + assert api.calls == [(eco, 1) for eco in s.ECOSYSTEMS] and resumed == rows class Bracket(FakeOSV): def get(self, identifier): @@ -599,15 +603,75 @@ def test_reduced_advisories_preserve_all_fix_exclusions_and_selection(): s.candidate(s.reduce_advisory(raw), FakeAPI(), FakeOSV()) -def test_old_raw_advisory_journal_version_refused_before_calls(tmp_path): +@pytest.mark.parametrize("old_version", [1, 2]) +def test_old_advisory_journal_version_refused_before_calls(tmp_path, old_version): used, api, osv, clones = e.UsedSet(), FakeAPI(1), FakeOSV(), FakeClones() path = tmp_path / "journal" parameters = d.inputs(used, 7, set(used.sources), clones) - assert parameters["journal_version"] == 2 - with d.Journal(path).open({**parameters, "journal_version": 1}) as journal: + assert parameters["journal_version"] == 3 + with d.Journal(path).open({**parameters, "journal_version": old_version}) as journal: journal.put("advisories", "1", {"rows": list(api.advisories()), "done": True}) report = d.run( tmp_path / "repo", github=api, osv=osv, clones=clones, used=used, seed=7, known_empty=set(used.sources), journal_path=path ) assert report["status"] == "resume_refused" and report["changed_parameters"] == ["journal_version"] assert api.calls == osv.calls == 0 and clones.counts == {} + + +def test_enumerate_ecosystem_pages_dedupes_and_resumes_partial_stream(tmp_path): + rows = list(FakeAPI(3).advisories()) + pages = { + ("pip", 1): {"rows": [rows[0]], "done": False}, + ("pip", 2): {"rows": [rows[1]], "done": True}, + ("npm", 1): {"rows": [rows[0], rows[2]], "done": True}, + ("maven", 1): {"rows": [], "done": True}, + ("composer", 1): {"rows": [], "done": True}, + } + calls = [] + + class Pages: + def advisory_page(self, page, *, ecosystem): + calls.append((ecosystem, page)) + return pages[ecosystem, page] + + path = tmp_path / "journal" + with d.Journal(path).open({"journal_version": 3}) as journal: + journal.put("advisories", "pip:1", pages["pip", 1]) + with d.Journal(path).open({"journal_version": 3}) as journal: + result = d.enumerate_advisories(Pages(), journal) + assert [row["ghsa_id"] for row in result] == [row["ghsa_id"] for row in rows] + assert list(journal.rows["advisories"]) == [f"{eco}:{page}" for eco, page in pages] + assert calls == list(pages)[1:] + with d.Journal(path).open({"journal_version": 3}) as journal: + assert d.enumerate_advisories(Pages(), journal) == result + assert calls == list(pages)[1:] + + +def test_filtered_http_population_reaches_draft_floor_with_bounded_pages(tmp_path, monkeypatch): + from urllib.parse import parse_qs, urlsplit + + population = list(FakeAPI(300).advisories()) + pages = [] + + class FilteredTransport: + def get(self, url, headers): + query = parse_qs(urlsplit(url).query) + # An unfiltered request would enumerate the global database: fail immediately. + assert query["ecosystem"][0] in s.ECOSYSTEMS + assert query["published"] == [">=2019-01-01"] + assert "89" in query["cwes"][0].split(",") + eco, page = query["ecosystem"][0], int(query["page"][0]) + pages.append((eco, page)) + assert len(pages) <= 7 + rows = population if eco == "pip" else [] + return rows[(page - 1) * 100 : page * 100] + + api, used = FakeAPI(), e.UsedSet() + http = s.GitHub("fake", FilteredTransport()) + monkeypatch.setattr(api, "advisory_page", http.advisory_page) + changes = entries()[0]["changes"] + monkeypatch.setattr(d, "repository_changes", lambda *a, **kw: (changes, {}, {"bytes": 12, "files": 1})) + report = d.run(tmp_path, github=api, osv=FakeOSV(), clones=FakeClones(), used=used, seed=7, known_empty=set(used.sources)) + assert report["status"] == "draft" and report["repositories"] == 300 + assert report["advisories"] == 300 + assert pages == [("pip", n) for n in range(1, 5)] + [(eco, 1) for eco in s.ECOSYSTEMS[1:]] diff --git a/tests/test_unseen_source.py b/tests/test_unseen_source.py index 37804b5..49b77bb 100644 --- a/tests/test_unseen_source.py +++ b/tests/test_unseen_source.py @@ -122,7 +122,9 @@ def test_http_adapters_with_fake_transport(): parsed = urlsplit(url) if parsed.path == "/advisories": page = int(parse_qs(parsed.query)["page"][0]) - return [advisory()] * (100 if page == 1 else 1) + return [ + {**advisory(), "ghsa_id": f"GHSA-test-test-{i:04d}"} for i in range(0 if page == 1 else 100, 100 if page == 1 else 101) + ] if "/license" in parsed.path: assert parse_qs(parsed.query)["ref"] == ["b" * 40] return { @@ -271,3 +273,55 @@ def test_transport_decode_retry_respects_response_retry_after(): transport = s.JSONTransport(opener=lambda *a, **kw: responses.pop(0), sleeper=waits.append) assert transport.get("https://example.invalid", {}) == {} assert waits == [9] + + +@pytest.mark.parametrize("ecosystem", s.ECOSYSTEMS) +def test_advisory_page_filters_match_candidate_population(ecosystem): + from urllib.parse import parse_qs, urlsplit + + taxonomy = s.load_families() + expected = {c for f in taxonomy.families if f.id in s.FAMILIES for c in f.cwes} + + class Transport: + def get(self, url, headers): + query = parse_qs(urlsplit(url).query) + assert query == { + "type": ["reviewed"], + "per_page": ["100"], + "page": ["2"], + "sort": ["published"], + "direction": ["asc"], + "ecosystem": [ecosystem], + "published": [">=2019-01-01"], + "cwes": [",".join(sorted((c.removeprefix("CWE-") for c in expected), key=int))], + } + return [] + + assert s.GitHub("fake", Transport()).advisory_page(2, ecosystem=ecosystem) == {"rows": [], "done": True} + # Every enumerated CWE is selectable at the inclusive date boundary. + for cwe in expected: + row = advisory() + row.update(published_at="2019-01-01T00:00:00Z", cwes=[{"cwe_id": cwe}]) + row["vulnerabilities"][0]["package"]["ecosystem"] = ecosystem + assert s.candidate(row, API(), OSV()).family == taxonomy.family_of_cwe(cwe).id + + +@pytest.mark.parametrize("token", ["CWE-89*", "CWE-", "89", "CWE-79..CWE-89"]) +def test_advisory_page_omits_entire_cwe_filter_for_non_exact_taxonomy(monkeypatch, token): + from dataclasses import replace + from urllib.parse import parse_qs, urlsplit + + taxonomy = s.load_families() + family = taxonomy.by_id("injection") + taxonomy = replace(taxonomy, families=tuple(replace(f, cwes=f.cwes | {token}) if f == family else f for f in taxonomy.families)) + monkeypatch.setattr(s, "load_families", lambda: taxonomy) + + class Transport: + def get(self, url, headers): + query = parse_qs(urlsplit(url).query) + assert "cwes" not in query + assert query["published"] == [">=2019-01-01"] + assert query["ecosystem"] == ["pip"] + return [] + + s.GitHub("fake", Transport()).advisory_page(1, ecosystem="pip")