unseen: filter advisory enumeration server-side (ecosystem/date/cwes)

enumerate_advisories paged the entire global GitHub reviewed-advisory
database (every ecosystem, every year, every CWE) before extraction, so
a 300-repo draft never converged -- a live run reached 21,962 advisories
in 4.5h with extraction never started.

Push the existing candidate() rejections into the query: page per
ecosystem, published>=2019-01-01, and cwes restricted to the seven
families. The cwes filter uses bare CWE numbers (89), which is what the
GitHub advisories API matches -- "CWE-89" silently returns zero. The
whole cwes filter is omitted if the taxonomy ever holds a non-exact
token, so a selectable advisory is never dropped. Journal version 3
(ecosystem:page keys) refuses incompatible older journals.

Live page-1 smoke (pip, npm): 100/100 rows carry a family CWE, none
published before 2019. Offline: 166 unseen tests pass, ruff clean.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

benchmarks/unseen/draft.py

@@ -174,7 +174,7 @@ def prove_source(git, revision: str) -> dict:
 
 
 def advisory_index(rows: list[dict], github) -> dict[str, set[str]]:
-    """Include ALL enumerated fixes, even old, unmapped, multi-fix and other ecosystems.
+    """Include ALL fixes in the enumerated population, even from rejected candidates.
 
     Canonicalize link names as well, so an old advisory URL survives repository renames.
     A 404 is not treated as proof that an advisory is unrelated to a selected repository.
@@ -219,7 +219,7 @@ def inputs(used, seed, known_empty, clones):
         "used_set_digest": digest({label: sorted(names) for label, names in used.sources.items()}),
         "known_empty": sorted(known_empty),
         "clone_limit_bytes": getattr(clones, "limit_bytes", 500 * 1024 * 1024),
-        "journal_version": 2,
+        "journal_version": 3,
     }
     # Hardcoded design thresholds and algorithms are covered as well as taxonomy data.
     for name in ("draft", "source", "extract", "eligibility", "journal"):
@@ -229,18 +229,20 @@ def inputs(used, seed, known_empty, clones):
 
 
 def enumerate_advisories(github, journal):
-    page, records = 1, {}
-    while True:
-        key = str(page)
-        if key not in journal.rows["advisories"]:
-            journal.boundary()
-            result = github.advisory_page(page)
-            journal.put("advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "done": result["done"]})
-        result = journal.rows["advisories"][key]
-        records.update((row["ghsa_id"], row) for row in result["rows"])
-        if result["done"]:
-            break
-        page += 1
+    records = {}
+    for ecosystem in source.ECOSYSTEMS:
+        page = 1
+        while True:
+            key = f"{ecosystem}:{page}"
+            if key not in journal.rows["advisories"]:
+                journal.boundary()
+                result = github.advisory_page(page, ecosystem=ecosystem)
+                journal.put("advisories", key, {"rows": [source.reduce_advisory(row) for row in result["rows"]], "done": result["done"]})
+            result = journal.rows["advisories"][key]
+            records.update((row["ghsa_id"], row) for row in result["rows"])
+            if result["done"]:
+                break
+            page += 1
     if not records:
         raise InstrumentFailure("empty_advisories")
     return list(records.values())

benchmarks/unseen/source.py

@@ -269,24 +269,46 @@ class GitHub:
             "text": base64.b64decode(row.get("content", "")).decode("utf-8", errors="replace"),
         }
 
-    def advisory_page(self, page: int) -> dict:
+    def advisory_page(self, page: int, *, ecosystem: str) -> dict:
+        if ecosystem not in ECOSYSTEMS:
+            raise ValueError("invalid_ecosystem")
         # Oldest first: newly published advisories do not shift already persisted pages.
-        rows = self.get(
-            "/advisories?" + urlencode({"type": "reviewed", "per_page": 100, "page": page, "sort": "published", "direction": "asc"})
-        )
+        parameters = {
+            "type": "reviewed",
+            "per_page": 100,
+            "page": page,
+            "sort": "published",
+            "direction": "asc",
+            "ecosystem": ecosystem,
+            # Inclusive syntax used in benchmarks/independent/research-2026-09-30-php.md.
+            "published": ">=2019-01-01",
+        }
+        taxonomy = load_families()
+        cwes = {cwe for family in taxonomy.families if family.id in FAMILIES for cwe in family.cwes}
+        # family_of_cwe uses exact membership. If the taxonomy gains non-exact tokens,
+        # fetch broadly rather than risk excluding a client-selectable advisory.
+        # GitHub's advisories `cwes` filter expects bare numbers (89), not "CWE-89".
+        if cwes and all(re.fullmatch(r"CWE-[1-9][0-9]*", cwe) for cwe in cwes):
+            parameters["cwes"] = ",".join(sorted((cwe.removeprefix("CWE-") for cwe in cwes), key=int))
+        rows = self.get("/advisories?" + urlencode(parameters))
         if not isinstance(rows, list):
             raise ValueError("invalid_advisory_page")
         return {"rows": rows, "done": len(rows) < 100}
 
     def advisories(self):
-        # Enumerate every reviewed advisory, including non-candidate fixes.
-        page = 1
-        while True:
-            result = self.advisory_page(page)
-            yield from result["rows"]
-            if result["done"]:
-                break
-            page += 1
+        """Enumerate the candidate population, deduplicating multi-ecosystem advisories."""
+        seen = set()
+        for ecosystem in ECOSYSTEMS:
+            page = 1
+            while True:
+                result = self.advisory_page(page, ecosystem=ecosystem)
+                for row in result["rows"]:
+                    if row["ghsa_id"] not in seen:
+                        seen.add(row["ghsa_id"])
+                        yield row
+                if result["done"]:
+                    break
+                page += 1
 
 
 class OSV: