Merge unseen-sourcing-filter: server-side advisory filtering

Co-Authored-By: Claude Opus 4.8 <[email protected]>

benchmarks/unseen/draft.py

@@ -174,7 +174,7 @@ def prove_source(git, revision: str) -> dict:
 
 
 def advisory_index(rows: list[dict], github) -> dict[str, set[str]]:
-    """Include ALL enumerated fixes, even old, unmapped, multi-fix and other ecosystems.
+    """Include ALL fixes in the enumerated population, even from rejected candidates.
 
     Canonicalize link names as well, so an old advisory URL survives repository renames.
     A 404 is not treated as proof that an advisory is unrelated to a selected repository.
@@ -219,7 +219,7 @@ def inputs(used, seed, known_empty, clones):
         "used_set_digest": digest({label: sorted(names) for label, names in used.sources.items()}),
         "known_empty": sorted(known_empty),
         "clone_limit_bytes": getattr(clones, "limit_bytes", 500 * 1024 * 1024),
-        "journal_version": 2,
+        "journal_version": 3,
     }
     # Hardcoded design thresholds and algorithms are covered as well as taxonomy data.
     for name in ("draft", "source", "extract", "eligibility", "journal"):
@@ -229,18 +229,20 @@ def inputs(used, seed, known_empty, clones):
 
 
 def enumerate_advisories(github, journal):
-    page, records = 1, {}
-    while True:
-        key = str(page)
-        if key not in journal.rows["advisories"]:
-            journal.boundary()
-            result = github.advisory_page(page)
-            journal.put("advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "done": result["done"]})
-        result = journal.rows["advisories"][key]
-        records.update((row["ghsa_id"], row) for row in result["rows"])
-        if result["done"]:
-            break
-        page += 1
+    records = {}
+    for ecosystem in source.ECOSYSTEMS:
+        page = 1
+        while True:
+            key = f"{ecosystem}:{page}"
+            if key not in journal.rows["advisories"]:
+                journal.boundary()
+                result = github.advisory_page(page, ecosystem=ecosystem)
+                journal.put("advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "done": result["done"]})
+            result = journal.rows["advisories"][key]
+            records.update((row["ghsa_id"], row) for row in result["rows"])
+            if result["done"]:
+                break
+            page += 1
     if not records:
         raise InstrumentFailure("empty_advisories")
     return list(records.values())

benchmarks/unseen/source.py

@@ -269,24 +269,46 @@ class GitHub:
             "text": base64.b64decode(row.get("content", "")).decode("utf-8", errors="replace"),
         }
 
-    def advisory_page(self, page: int) -> dict:
+    def advisory_page(self, page: int, *, ecosystem: str) -> dict:
+        if ecosystem not in ECOSYSTEMS:
+            raise ValueError("invalid_ecosystem")
         # Oldest first: newly published advisories do not shift already persisted pages.
-        rows = self.get(
-            "/advisories?" + urlencode({"type": "reviewed", "per_page": 100, "page": page, "sort": "published", "direction": "asc"})
-        )
+        parameters = {
+            "type": "reviewed",
+            "per_page": 100,
+            "page": page,
+            "sort": "published",
+            "direction": "asc",
+            "ecosystem": ecosystem,
+            # Inclusive syntax used in benchmarks/independent/research-2026-09-30-php.md.
+            "published": ">=2019-01-01",
+        }
+        taxonomy = load_families()
+        cwes = {cwe for family in taxonomy.families if family.id in FAMILIES for cwe in family.cwes}
+        # family_of_cwe uses exact membership. If the taxonomy gains non-exact tokens,
+        # fetch broadly rather than risk excluding a client-selectable advisory.
+        # GitHub's advisories `cwes` filter expects bare numbers (89), not "CWE-89".
+        if cwes and all(re.fullmatch(r"CWE-[1-9][0-9]*", cwe) for cwe in cwes):
+            parameters["cwes"] = ",".join(sorted((cwe.removeprefix("CWE-") for cwe in cwes), key=int))
+        rows = self.get("/advisories?" + urlencode(parameters))
         if not isinstance(rows, list):
             raise ValueError("invalid_advisory_page")
         return {"rows": rows, "done": len(rows) < 100}
 
     def advisories(self):
-        # Enumerate every reviewed advisory, including non-candidate fixes.
-        page = 1
-        while True:
-            result = self.advisory_page(page)
-            yield from result["rows"]
-            if result["done"]:
-                break
-            page += 1
+        """Enumerate the candidate population, deduplicating multi-ecosystem advisories."""
+        seen = set()
+        for ecosystem in ECOSYSTEMS:
+            page = 1
+            while True:
+                result = self.advisory_page(page, ecosystem=ecosystem)
+                for row in result["rows"]:
+                    if row["ghsa_id"] not in seen:
+                        seen.add(row["ghsa_id"])
+                        yield row
+                if result["done"]:
+                    break
+                page += 1
 
 
 class OSV:

tests/test_unseen_draft.py

@@ -118,6 +118,7 @@ class FakeAPI:
     def __init__(self, count=300):
         self.count = count
         self.licenses = []
+        self.advisory_pages = []
 
     def get_repo(self, name):
         self.calls += 1
@@ -138,9 +139,11 @@ class FakeAPI:
         # P is MIT but actual introducing head is GPL: public output must use V.
         return {"text": "", "spdx_id": "MIT" if ref == "b" * 40 else "GPL-3.0", "path": "LICENSE"}
 
-    def advisory_page(self, page):
+    def advisory_page(self, page, *, ecosystem):
         self.calls += 1
-        rows = list(self.advisories())[(page - 1) * 100 : page * 100]
+        self.advisory_pages.append((ecosystem, page))
+        population = list(self.advisories()) if ecosystem == "pip" else []
+        rows = population[(page - 1) * 100 : page * 100]
         return {"rows": rows, "done": len(rows) < 100}
 
     def advisories(self):
@@ -457,8 +460,8 @@ def test_max_hours_stops_after_completed_page(tmp_path):
     now = [0.0]
 
     class TimedAPI(FakeAPI):
-        def advisory_page(self, page):
-            result = super().advisory_page(page)
+        def advisory_page(self, page, *, ecosystem):
+            result = super().advisory_page(page, ecosystem=ecosystem)
             now[0] += 3601
             return result
 
@@ -541,8 +544,8 @@ def test_advisory_journal_reduces_nested_blobs_and_resumes_by_page(tmp_path):
     class Pages:
         calls = []
 
-        def advisory_page(self, page):
-            self.calls.append(page)
+        def advisory_page(self, page, *, ecosystem):
+            self.calls.append((ecosystem, page))
             return {"rows": [raw], "done": True, "unused": "raw page blob"}
 
     api, used = Pages(), e.UsedSet()
@@ -551,14 +554,15 @@ def test_advisory_journal_reduces_nested_blobs_and_resumes_by_page(tmp_path):
     with d.Journal(path).open(parameters) as journal:
         rows = d.enumerate_advisories(api, journal)
         assert rows == [expected]
-    saved = json.loads((path / "advisories.jsonl").read_bytes())
-    assert saved["value"] == {"rows": [expected], "done": True}
+    saved = [json.loads(line) for line in (path / "advisories.jsonl").read_bytes().splitlines()]
+    assert [row["id"] for row in saved] == [f"{eco}:1" for eco in s.ECOSYSTEMS]
+    assert all(row["value"] == {"rows": [expected], "done": True} for row in saved)
     assert b"unused" not in (path / "advisories.jsonl").read_bytes()
     assert b"raw advisory blob" not in (path / "advisories.jsonl").read_bytes()
     assert b"raw page blob" not in (path / "advisories.jsonl").read_bytes()
     with d.Journal(path).open(parameters) as journal:
         resumed = d.enumerate_advisories(api, journal)
-    assert api.calls == [1] and resumed == rows
+    assert api.calls == [(eco, 1) for eco in s.ECOSYSTEMS] and resumed == rows
 
     class Bracket(FakeOSV):
         def get(self, identifier):
@@ -599,15 +603,75 @@ def test_reduced_advisories_preserve_all_fix_exclusions_and_selection():
             s.candidate(s.reduce_advisory(raw), FakeAPI(), FakeOSV())
 
 
-def test_old_raw_advisory_journal_version_refused_before_calls(tmp_path):
+@pytest.mark.parametrize("old_version", [1, 2])
+def test_old_advisory_journal_version_refused_before_calls(tmp_path, old_version):
     used, api, osv, clones = e.UsedSet(), FakeAPI(1), FakeOSV(), FakeClones()
     path = tmp_path / "journal"
     parameters = d.inputs(used, 7, set(used.sources), clones)
-    assert parameters["journal_version"] == 2
-    with d.Journal(path).open({**parameters, "journal_version": 1}) as journal:
+    assert parameters["journal_version"] == 3
+    with d.Journal(path).open({**parameters, "journal_version": old_version}) as journal:
         journal.put("advisories", "1", {"rows": list(api.advisories()), "done": True})
     report = d.run(
         tmp_path / "repo", github=api, osv=osv, clones=clones, used=used, seed=7, known_empty=set(used.sources), journal_path=path
     )
     assert report["status"] == "resume_refused" and report["changed_parameters"] == ["journal_version"]
     assert api.calls == osv.calls == 0 and clones.counts == {}
+
+
+def test_enumerate_ecosystem_pages_dedupes_and_resumes_partial_stream(tmp_path):
+    rows = list(FakeAPI(3).advisories())
+    pages = {
+        ("pip", 1): {"rows": [rows[0]], "done": False},
+        ("pip", 2): {"rows": [rows[1]], "done": True},
+        ("npm", 1): {"rows": [rows[0], rows[2]], "done": True},
+        ("maven", 1): {"rows": [], "done": True},
+        ("composer", 1): {"rows": [], "done": True},
+    }
+    calls = []
+
+    class Pages:
+        def advisory_page(self, page, *, ecosystem):
+            calls.append((ecosystem, page))
+            return pages[ecosystem, page]
+
+    path = tmp_path / "journal"
+    with d.Journal(path).open({"journal_version": 3}) as journal:
+        journal.put("advisories", "pip:1", pages["pip", 1])
+    with d.Journal(path).open({"journal_version": 3}) as journal:
+        result = d.enumerate_advisories(Pages(), journal)
+        assert [row["ghsa_id"] for row in result] == [row["ghsa_id"] for row in rows]
+        assert list(journal.rows["advisories"]) == [f"{eco}:{page}" for eco, page in pages]
+    assert calls == list(pages)[1:]
+    with d.Journal(path).open({"journal_version": 3}) as journal:
+        assert d.enumerate_advisories(Pages(), journal) == result
+    assert calls == list(pages)[1:]
+
+
+def test_filtered_http_population_reaches_draft_floor_with_bounded_pages(tmp_path, monkeypatch):
+    from urllib.parse import parse_qs, urlsplit
+
+    population = list(FakeAPI(300).advisories())
+    pages = []
+
+    class FilteredTransport:
+        def get(self, url, headers):
+            query = parse_qs(urlsplit(url).query)
+            # An unfiltered request would enumerate the global database: fail immediately.
+            assert query["ecosystem"][0] in s.ECOSYSTEMS
+            assert query["published"] == [">=2019-01-01"]
+            assert "89" in query["cwes"][0].split(",")
+            eco, page = query["ecosystem"][0], int(query["page"][0])
+            pages.append((eco, page))
+            assert len(pages) <= 7
+            rows = population if eco == "pip" else []
+            return rows[(page - 1) * 100 : page * 100]
+
+    api, used = FakeAPI(), e.UsedSet()
+    http = s.GitHub("fake", FilteredTransport())
+    monkeypatch.setattr(api, "advisory_page", http.advisory_page)
+    changes = entries()[0]["changes"]
+    monkeypatch.setattr(d, "repository_changes", lambda *a, **kw: (changes, {}, {"bytes": 12, "files": 1}))
+    report = d.run(tmp_path, github=api, osv=FakeOSV(), clones=FakeClones(), used=used, seed=7, known_empty=set(used.sources))
+    assert report["status"] == "draft" and report["repositories"] == 300
+    assert report["advisories"] == 300
+    assert pages == [("pip", n) for n in range(1, 5)] + [(eco, 1) for eco in s.ECOSYSTEMS[1:]]

tests/test_unseen_source.py

@@ -122,7 +122,9 @@ def test_http_adapters_with_fake_transport():
             parsed = urlsplit(url)
             if parsed.path == "/advisories":
                 page = int(parse_qs(parsed.query)["page"][0])
-                return [advisory()] * (100 if page == 1 else 1)
+                return [
+                    {**advisory(), "ghsa_id": f"GHSA-test-test-{i:04d}"} for i in range(0 if page == 1 else 100, 100 if page == 1 else 101)
+                ]
             if "/license" in parsed.path:
                 assert parse_qs(parsed.query)["ref"] == ["b" * 40]
                 return {
@@ -271,3 +273,55 @@ def test_transport_decode_retry_respects_response_retry_after():
     transport = s.JSONTransport(opener=lambda *a, **kw: responses.pop(0), sleeper=waits.append)
     assert transport.get("https://example.invalid", {}) == {}
     assert waits == [9]
+
+
+@pytest.mark.parametrize("ecosystem", s.ECOSYSTEMS)
+def test_advisory_page_filters_match_candidate_population(ecosystem):
+    from urllib.parse import parse_qs, urlsplit
+
+    taxonomy = s.load_families()
+    expected = {c for f in taxonomy.families if f.id in s.FAMILIES for c in f.cwes}
+
+    class Transport:
+        def get(self, url, headers):
+            query = parse_qs(urlsplit(url).query)
+            assert query == {
+                "type": ["reviewed"],
+                "per_page": ["100"],
+                "page": ["2"],
+                "sort": ["published"],
+                "direction": ["asc"],
+                "ecosystem": [ecosystem],
+                "published": [">=2019-01-01"],
+                "cwes": [",".join(sorted((c.removeprefix("CWE-") for c in expected), key=int))],
+            }
+            return []
+
+    assert s.GitHub("fake", Transport()).advisory_page(2, ecosystem=ecosystem) == {"rows": [], "done": True}
+    # Every enumerated CWE is selectable at the inclusive date boundary.
+    for cwe in expected:
+        row = advisory()
+        row.update(published_at="2019-01-01T00:00:00Z", cwes=[{"cwe_id": cwe}])
+        row["vulnerabilities"][0]["package"]["ecosystem"] = ecosystem
+        assert s.candidate(row, API(), OSV()).family == taxonomy.family_of_cwe(cwe).id
+
+
+@pytest.mark.parametrize("token", ["CWE-89*", "CWE-", "89", "CWE-79..CWE-89"])
+def test_advisory_page_omits_entire_cwe_filter_for_non_exact_taxonomy(monkeypatch, token):
+    from dataclasses import replace
+    from urllib.parse import parse_qs, urlsplit
+
+    taxonomy = s.load_families()
+    family = taxonomy.by_id("injection")
+    taxonomy = replace(taxonomy, families=tuple(replace(f, cwes=f.cwes | {token}) if f == family else f for f in taxonomy.families))
+    monkeypatch.setattr(s, "load_families", lambda: taxonomy)
+
+    class Transport:
+        def get(self, url, headers):
+            query = parse_qs(urlsplit(url).query)
+            assert "cwes" not in query
+            assert query["published"] == [">=2019-01-01"]
+            assert query["ecosystem"] == ["pip"]
+            return []
+
+    s.GitHub("fake", Transport()).advisory_page(1, ecosystem="pip")