"""Advisory selection. Network adapters are constructed only by the coordinator CLI.""" from __future__ import annotations import base64 import json import random import re from collections import OrderedDict from dataclasses import dataclass from datetime import date from urllib.parse import urlencode, urlsplit from openultrasast.model.taxonomy import load_families from .eligibility import RepositoryNotFound, normalize ECOSYSTEMS = ("pip", "npm", "maven", "composer") OSV_ECOSYSTEMS = dict(zip(ECOSYSTEMS, ("PyPI", "npm", "Maven", "Packagist"), strict=True)) FAMILIES = {"injection", "path", "deserialization", "access_control", "output_encoding", "untrusted_destination", "config_secrets"} PERMISSIVE = {"MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", "0BSD", "Unlicense", "Zlib", "PostgreSQL", "BSL-1.0"} FIX = re.compile(r"https://github\.com/([\w.-]+/[\w.-]+)/commit/([a-fA-F0-9]{7,40})(?:[/?#].*)?$", re.I) class Rejected(ValueError): """Expected eligibility rejection; messages are fixed reason codes, never names.""" @dataclass(frozen=True) class Candidate: repository: str url: str fix: str parent: str advisory: str ecosystem: str family: str post_cutoff: bool first_affected: str | None license_spdx: str license_class: str license_ref: str license_path: str def compact_advisory(row: dict) -> dict: """Keep exactly the input fields used by selection and the all-advisory fix index.""" return { **{key: row[key] for key in ("ghsa_id", "published_at", "type", "withdrawn_at") if key in row}, "cwes": [{"cwe_id": c["cwe_id"]} for c in row.get("cwes", [])], "references": [f"https://github.com/{name}/commit/{sha}" for name, sha in sorted(fix_links(row))], "vulnerabilities": [{"package": {key: v["package"][key] for key in ("ecosystem", "name")}} for v in row.get("vulnerabilities", [])], } def compact_repository(row: dict) -> dict: return { **{key: row[key] for key in ("full_name", "private", "size", "clone_url", "former_names") if key in row}, **{key: {"full_name": row[key]["full_name"]} for key in ("parent", "source") if row.get(key)}, } def post_cutoff(row: dict) -> bool: return date.fromisoformat(row["published_at"][:10]) > date(2026, 6, 1) def fix_links(row: dict) -> set[tuple[str, str]]: links = set() for reference in row.get("references", []): url = reference if isinstance(reference, str) else reference.get("url", "") match = FIX.fullmatch(url) if match: links.add((normalize(match[1]), match[2].lower())) return links def ecosystem(row: dict) -> str | None: values = {v["package"]["ecosystem"].lower() for v in row.get("vulnerabilities", [])} return next((e for e in ECOSYSTEMS if e in values), None) def ordered_advisories(rows: list[dict], seed: int) -> list[dict]: """Stable seeded ordering, post-cutoff first in each ecosystem; round-robin strata.""" from .storage import AdvisoryRows if isinstance(rows, AdvisoryRows): return rows.ordered(seed) rng = random.Random(seed) groups = [] for eco in ECOSYSTEMS: group = sorted((r for r in rows if ecosystem(r) == eco), key=lambda r: r["ghsa_id"]) rng.shuffle(group) group.sort(key=lambda r: not post_cutoff(r)) groups.append(group) return [group[i] for i in range(max(map(len, groups), default=0)) for group in groups if i < len(group)] def first_affected(record: dict, row: dict, eco: str) -> str | None: packages = {v["package"]["name"] for v in row["vulnerabilities"] if v["package"]["ecosystem"].lower() == eco} versions = set() for affected in record.get("affected", []): package = affected.get("package", {}) if package.get("ecosystem") != OSV_ECOSYSTEMS[eco] or package.get("name") not in packages: continue for span in affected.get("ranges", []): if span.get("type") not in {"ECOSYSTEM", "SEMVER"}: continue for event in span.get("events", []): if event.get("introduced") not in {None, "0"}: versions.add(event["introduced"]) if len(versions) > 1: raise Rejected("ambiguous_version_bracket") return next(iter(versions), None) def license_info(record: dict) -> tuple[str, str]: # Explicit pinned-file SPDX wins. For GitHub-recognized full texts its license endpoint # provides the detected SPDX. Unknown/custom/compound declarations stay private. declared = re.search(r"SPDX-License-Identifier:\s*([^\r\n]+)", record.get("text", "")) spdx = declared[1].strip().removesuffix("*/").strip() if declared else record.get("spdx_id") or "NONE" return spdx, "permissive" if spdx in PERMISSIVE else "private" def candidate(row: dict, github, osv) -> Candidate: if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1): raise Rejected("date") if row.get("type", "reviewed") != "reviewed" or row.get("withdrawn_at"): raise Rejected("advisory_status") eco = ecosystem(row) if eco is None: raise Rejected("ecosystem") taxonomy = load_families() families = {f.id for c in row.get("cwes", []) if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in FAMILIES} if len(families) != 1: raise Rejected("family") links = fix_links(row) if len(links) != 1: raise Rejected("fix_links") name, fix = next(iter(links)) repo = github.get_repo(name) url = urlsplit(repo.get("clone_url", "")) if ( repo.get("private", True) or not 0 <= repo.get("size", -1) <= 500 * 1024 or url.scheme != "https" or url.hostname != "github.com" or url.username or url.password or url.query or url.fragment or normalize(repo["clone_url"]) != normalize(repo["full_name"]) ): raise Rejected("repository") commit = github.get_commit(repo["full_name"], fix) if len(commit["parents"]) != 1: raise Rejected("parents") parent = commit["parents"][0]["sha"] license_record = github.get_license(repo["full_name"], parent) spdx, license_class = license_info(license_record) return Candidate( normalize(repo["full_name"]), repo["clone_url"], commit["sha"], parent, row["ghsa_id"], eco, next(iter(families)), post_cutoff(row), first_affected(osv.get(row["ghsa_id"]), row, eco), spdx, license_class, parent, license_record.get("path", ""), ) class JSONTransport: """Coordinator GET transport with bounded retries; all I/O and waits are injectable.""" def __init__(self, *, opener=None, sleeper=None, clock=None): import time from urllib.request import urlopen self.opener = opener or urlopen self.sleeper = sleeper or time.sleep self.clock = clock or time.time self.attempts = 0 self.rate_limit = {} def get(self, url: str, headers: dict) -> object: from urllib.error import HTTPError from urllib.request import Request for attempt in range(5): self.attempts += 1 try: with self.opener(Request(url, headers=headers), timeout=60) as response: self.record_rate(getattr(response, "headers", {})) return json.load(response) except HTTPError as exc: self.record_rate(exc.headers) if exc.code == 404: raise RepositoryNotFound("not_found") from None limited = exc.code == 429 or ( exc.code == 403 and (exc.headers.get("X-RateLimit-Remaining") == "0" or exc.headers.get("Retry-After")) ) transient = exc.code in {500, 502, 503, 504} if attempt == 4 or not (limited or transient): raise ValueError("http_failure") from None try: delay = float(exc.headers.get("Retry-After", "0")) if limited and not delay: delay = max(60, float(exc.headers.get("X-RateLimit-Reset", "0")) - self.clock() + 1) delay = max(delay, 2**attempt) if delay > 3605: raise ValueError("retry_window") except (TypeError, ValueError): raise ValueError("http_retry_failed") from None self.sleeper(delay) except Exception: raise ValueError("http_failure") from None raise ValueError("http_failure") def record_rate(self, headers): self.rate_limit = { key: headers.get(header) for key, header in ( ("remaining", "X-RateLimit-Remaining"), ("limit", "X-RateLimit-Limit"), ("reset", "X-RateLimit-Reset"), ("retry_after", "Retry-After"), ) if headers.get(header) is not None } class GitHub: def __init__(self, token: str, transport=None): if not token: raise ValueError("missing_gh_token") self.transport = transport or JSONTransport() self.headers = {"Authorization": "Bearer " + token, "Accept": "application/vnd.github+json"} self.repos: OrderedDict[str, dict] = OrderedDict() self.calls = 0 def get(self, path: str): self.calls += 1 return self.transport.get("https://api.github.com" + path, self.headers) def get_repo(self, name: str) -> dict: name = normalize(name) if name not in self.repos: self.repos[name] = compact_repository(self.get("/repos/" + name)) if len(self.repos) > 128: self.repos.popitem(last=False) self.repos.move_to_end(name) return self.repos[name] def get_commit(self, name: str, sha: str) -> dict: return self.get("/repos/" + normalize(name) + "/commits/" + sha) def get_license(self, name: str, ref: str) -> dict: try: row = self.get("/repos/" + normalize(name) + "/license?" + urlencode({"ref": ref})) except RepositoryNotFound: return {"spdx_id": "NONE", "text": "", "path": ""} return { "spdx_id": (row.get("license") or {}).get("spdx_id"), "path": row.get("path", ""), "text": base64.b64decode(row.get("content", "")).decode("utf-8", errors="replace"), } def advisory_page(self, page: int) -> dict: # Oldest first: newly published advisories do not shift already persisted pages. rows = self.get( "/advisories?" + urlencode({"type": "reviewed", "per_page": 100, "page": page, "sort": "published", "direction": "asc"}) ) if not isinstance(rows, list): raise ValueError("invalid_advisory_page") return {"rows": rows, "done": len(rows) < 100} def advisories(self): # Enumerate every reviewed advisory, including non-candidate fixes. page = 1 while True: result = self.advisory_page(page) yield from result["rows"] if result["done"]: break page += 1 class OSV: def __init__(self, transport=None): self.transport = transport or JSONTransport() self.calls = 0 def get(self, identifier: str) -> dict: if not re.fullmatch(r"GHSA-[\w-]+", identifier): raise ValueError("invalid_advisory_id") self.calls += 1 try: return self.transport.get("https://api.osv.dev/v1/vulns/" + identifier, {}) except RepositoryNotFound: return {} # No OSV bracket is distinct from a failed service call.