mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 4.8
5620165540
The GitHub /advisories endpoint ignores the `page` query parameter: it paginates with the Link header's rel="next" cursor. The page-number pager re-fetched the same first 100 rows forever and never terminated, so enumeration never completed and no draft converged (a run fetched 5137 "pip pages" holding only 100 unique advisories). Surface the successful response headers from JSONTransport, and walk the rel="next" cursor per ecosystem until it is absent. The ecosystem / published>=2019 / bare-number cwes filters are preserved on the first request. Journal keys stay ecosystem:n (now cursor-walk positions) with the saved next URL for resume; journal_version 4 refuses older journals. Live smoke via the production pager: 300 unique advisories pulled in 3 API calls, advancing past the first page. Offline: 170 unseen tests pass, ruff clean. Co-Authored-By: Claude Opus 4.8 <[email protected]>
332 lines
13 KiB
Python
332 lines
13 KiB
Python
"""Advisory selection. Network adapters are constructed only by the coordinator CLI."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import random
|
|
import re
|
|
from dataclasses import dataclass
|
|
from datetime import date
|
|
from urllib.parse import urlencode, urlsplit
|
|
|
|
from openultrasast.model.taxonomy import load_families
|
|
|
|
from .eligibility import RepositoryNotFound, normalize
|
|
|
|
ECOSYSTEMS = ("pip", "npm", "maven", "composer")
|
|
OSV_ECOSYSTEMS = dict(zip(ECOSYSTEMS, ("PyPI", "npm", "Maven", "Packagist"), strict=True))
|
|
FAMILIES = {"injection", "path", "deserialization", "access_control", "output_encoding", "untrusted_destination", "config_secrets"}
|
|
PERMISSIVE = {"MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", "0BSD", "Unlicense", "Zlib", "PostgreSQL", "BSL-1.0"}
|
|
FIX = re.compile(r"https://github\.com/([\w.-]+/[\w.-]+)/commit/([a-fA-F0-9]{7,40})(?:[/?#].*)?$", re.I)
|
|
|
|
|
|
class Rejected(ValueError):
|
|
"""Expected eligibility rejection; messages are fixed reason codes, never names."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Candidate:
|
|
repository: str
|
|
url: str
|
|
fix: str
|
|
parent: str
|
|
advisory: str
|
|
ecosystem: str
|
|
family: str
|
|
post_cutoff: bool
|
|
first_affected: str | None
|
|
license_spdx: str
|
|
license_class: str
|
|
license_ref: str
|
|
license_path: str
|
|
|
|
|
|
def post_cutoff(row: dict) -> bool:
|
|
return date.fromisoformat(row["published_at"][:10]) > date(2026, 6, 1)
|
|
|
|
|
|
def fix_links(row: dict) -> set[tuple[str, str]]:
|
|
links = set()
|
|
for reference in row.get("references", []):
|
|
url = reference if isinstance(reference, str) else reference.get("url", "")
|
|
match = FIX.fullmatch(url)
|
|
if match:
|
|
links.add((normalize(match[1]), match[2].lower()))
|
|
return links
|
|
|
|
|
|
def ecosystem(row: dict) -> str | None:
|
|
values = {v["package"]["ecosystem"].lower() for v in row.get("vulnerabilities", [])}
|
|
return next((e for e in ECOSYSTEMS if e in values), None)
|
|
|
|
|
|
def reduce_advisory(row: dict) -> dict:
|
|
"""Keep only selection inputs, including every fix needed by the exclusion index.
|
|
|
|
Preserve the parser's shape so fresh and resumed pages follow the same path.
|
|
Commit parents, licenses and OSV brackets are resolved later into Candidate.
|
|
"""
|
|
return {
|
|
"ghsa_id": row["ghsa_id"],
|
|
"published_at": row["published_at"],
|
|
"type": row.get("type", "reviewed"),
|
|
"withdrawn_at": row.get("withdrawn_at"),
|
|
"cwes": [{"cwe_id": c["cwe_id"]} for c in row.get("cwes", [])],
|
|
"vulnerabilities": [{"package": {key: v["package"][key] for key in ("ecosystem", "name")}} for v in row.get("vulnerabilities", [])],
|
|
"references": [f"https://github.com/{name}/commit/{sha}" for name, sha in sorted(fix_links(row))],
|
|
}
|
|
|
|
|
|
def ordered_advisories(rows: list[dict], seed: int) -> list[dict]:
|
|
"""Stable seeded ordering, post-cutoff first in each ecosystem; round-robin strata."""
|
|
rng = random.Random(seed)
|
|
groups = []
|
|
for eco in ECOSYSTEMS:
|
|
group = sorted((r for r in rows if ecosystem(r) == eco), key=lambda r: r["ghsa_id"])
|
|
rng.shuffle(group)
|
|
group.sort(key=lambda r: not post_cutoff(r))
|
|
groups.append(group)
|
|
return [group[i] for i in range(max(map(len, groups), default=0)) for group in groups if i < len(group)]
|
|
|
|
|
|
def first_affected(record: dict, row: dict, eco: str) -> str | None:
|
|
packages = {v["package"]["name"] for v in row["vulnerabilities"] if v["package"]["ecosystem"].lower() == eco}
|
|
versions = set()
|
|
for affected in record.get("affected", []):
|
|
package = affected.get("package", {})
|
|
if package.get("ecosystem") != OSV_ECOSYSTEMS[eco] or package.get("name") not in packages:
|
|
continue
|
|
for span in affected.get("ranges", []):
|
|
if span.get("type") not in {"ECOSYSTEM", "SEMVER"}:
|
|
continue
|
|
for event in span.get("events", []):
|
|
if event.get("introduced") not in {None, "0"}:
|
|
versions.add(event["introduced"])
|
|
if len(versions) > 1:
|
|
raise Rejected("ambiguous_version_bracket")
|
|
return next(iter(versions), None)
|
|
|
|
|
|
def license_info(record: dict) -> tuple[str, str]:
|
|
# Explicit pinned-file SPDX wins. For GitHub-recognized full texts its license endpoint
|
|
# provides the detected SPDX. Unknown/custom/compound declarations stay private.
|
|
declared = re.search(r"SPDX-License-Identifier:\s*([^\r\n]+)", record.get("text", ""))
|
|
spdx = declared[1].strip().removesuffix("*/").strip() if declared else record.get("spdx_id") or "NONE"
|
|
return spdx, "permissive" if spdx in PERMISSIVE else "private"
|
|
|
|
|
|
def candidate(row: dict, github, osv) -> Candidate:
|
|
if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1):
|
|
raise Rejected("date")
|
|
if row.get("type", "reviewed") != "reviewed" or row.get("withdrawn_at"):
|
|
raise Rejected("advisory_status")
|
|
eco = ecosystem(row)
|
|
if eco is None:
|
|
raise Rejected("ecosystem")
|
|
taxonomy = load_families()
|
|
families = {f.id for c in row.get("cwes", []) if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in FAMILIES}
|
|
if len(families) != 1:
|
|
raise Rejected("family")
|
|
links = fix_links(row)
|
|
if len(links) != 1:
|
|
raise Rejected("fix_links")
|
|
name, fix = next(iter(links))
|
|
repo = github.get_repo(name)
|
|
url = urlsplit(repo.get("clone_url", ""))
|
|
if (
|
|
repo.get("private", True)
|
|
or not 0 <= repo.get("size", -1) <= 500 * 1024
|
|
or url.scheme != "https"
|
|
or url.hostname != "github.com"
|
|
or url.username
|
|
or url.password
|
|
or url.query
|
|
or url.fragment
|
|
or normalize(repo["clone_url"]) != normalize(repo["full_name"])
|
|
):
|
|
raise Rejected("repository")
|
|
commit = github.get_commit(repo["full_name"], fix)
|
|
if len(commit["parents"]) != 1:
|
|
raise Rejected("parents")
|
|
parent = commit["parents"][0]["sha"]
|
|
license_record = github.get_license(repo["full_name"], parent)
|
|
spdx, license_class = license_info(license_record)
|
|
return Candidate(
|
|
normalize(repo["full_name"]),
|
|
repo["clone_url"],
|
|
commit["sha"],
|
|
parent,
|
|
row["ghsa_id"],
|
|
eco,
|
|
next(iter(families)),
|
|
post_cutoff(row),
|
|
first_affected(osv.get(row["ghsa_id"]), row, eco),
|
|
spdx,
|
|
license_class,
|
|
parent,
|
|
license_record.get("path", ""),
|
|
)
|
|
|
|
|
|
class JSONTransport:
|
|
"""Coordinator GET transport with bounded retries; all I/O and waits are injectable."""
|
|
|
|
def __init__(self, *, opener=None, sleeper=None, clock=None):
|
|
import time
|
|
from urllib.request import urlopen
|
|
|
|
self.opener = opener or urlopen
|
|
self.sleeper = sleeper or time.sleep
|
|
self.clock = clock or time.time
|
|
self.attempts = 0
|
|
self.rate_limit = {}
|
|
self.response_headers = {}
|
|
|
|
def get(self, url: str, headers: dict) -> object:
|
|
from http.client import IncompleteRead
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.request import Request
|
|
|
|
for attempt in range(5):
|
|
self.attempts += 1
|
|
retry_headers, limited = {}, False
|
|
try:
|
|
with self.opener(Request(url, headers=headers), timeout=60) as response:
|
|
retry_headers = getattr(response, "headers", {})
|
|
self.record_rate(retry_headers)
|
|
body = json.load(response)
|
|
self.response_headers = retry_headers
|
|
return body
|
|
except HTTPError as exc:
|
|
self.record_rate(exc.headers)
|
|
if exc.code == 404:
|
|
raise RepositoryNotFound("not_found") from None
|
|
limited = exc.code == 429 or (
|
|
exc.code == 403 and (exc.headers.get("X-RateLimit-Remaining") == "0" or exc.headers.get("Retry-After"))
|
|
)
|
|
transient = exc.code in {500, 502, 503, 504}
|
|
if attempt == 4 or not (limited or transient):
|
|
raise ValueError("http_failure") from None
|
|
retry_headers = exc.headers
|
|
except (URLError, TimeoutError, ConnectionError, IncompleteRead, json.JSONDecodeError):
|
|
if attempt == 4:
|
|
raise ValueError("http_failure") from None
|
|
except Exception:
|
|
raise ValueError("http_failure") from None
|
|
try:
|
|
delay = float(retry_headers.get("Retry-After", "0"))
|
|
if limited and not delay:
|
|
delay = max(60, float(retry_headers.get("X-RateLimit-Reset", "0")) - self.clock() + 1)
|
|
delay = max(delay, 2**attempt)
|
|
if delay > 3605:
|
|
raise ValueError("retry_window")
|
|
except (TypeError, ValueError):
|
|
raise ValueError("http_retry_failed") from None
|
|
self.sleeper(delay)
|
|
raise ValueError("http_failure")
|
|
|
|
def record_rate(self, headers):
|
|
self.rate_limit = {
|
|
key: headers.get(header)
|
|
for key, header in (
|
|
("remaining", "X-RateLimit-Remaining"),
|
|
("limit", "X-RateLimit-Limit"),
|
|
("reset", "X-RateLimit-Reset"),
|
|
("retry_after", "Retry-After"),
|
|
)
|
|
if headers.get(header) is not None
|
|
}
|
|
|
|
|
|
class GitHub:
|
|
def __init__(self, token: str, transport=None):
|
|
if not token:
|
|
raise ValueError("missing_gh_token")
|
|
self.transport = transport or JSONTransport()
|
|
self.headers = {"Authorization": "Bearer " + token, "Accept": "application/vnd.github+json"}
|
|
self.repos: dict[str, dict] = {}
|
|
self.calls = 0
|
|
|
|
def get(self, path: str):
|
|
self.calls += 1
|
|
url = path if path.startswith("https://api.github.com/") else "https://api.github.com" + path
|
|
return self.transport.get(url, self.headers)
|
|
|
|
def get_repo(self, name: str) -> dict:
|
|
name = normalize(name)
|
|
if name not in self.repos:
|
|
self.repos[name] = self.get("/repos/" + name)
|
|
return self.repos[name]
|
|
|
|
def get_commit(self, name: str, sha: str) -> dict:
|
|
return self.get("/repos/" + normalize(name) + "/commits/" + sha)
|
|
|
|
def get_license(self, name: str, ref: str) -> dict:
|
|
try:
|
|
row = self.get("/repos/" + normalize(name) + "/license?" + urlencode({"ref": ref}))
|
|
except RepositoryNotFound:
|
|
return {"spdx_id": "NONE", "text": "", "path": ""}
|
|
return {
|
|
"spdx_id": (row.get("license") or {}).get("spdx_id"),
|
|
"path": row.get("path", ""),
|
|
"text": base64.b64decode(row.get("content", "")).decode("utf-8", errors="replace"),
|
|
}
|
|
|
|
def advisory_page(self, next_url: str | None = None, *, ecosystem: str) -> dict:
|
|
if ecosystem not in ECOSYSTEMS:
|
|
raise ValueError("invalid_ecosystem")
|
|
# Oldest first: newly published advisories do not shift already persisted pages.
|
|
parameters = {
|
|
"type": "reviewed",
|
|
"per_page": 100,
|
|
"sort": "published",
|
|
"direction": "asc",
|
|
"ecosystem": ecosystem,
|
|
# Inclusive syntax used in benchmarks/independent/research-2026-09-30-php.md.
|
|
"published": ">=2019-01-01",
|
|
}
|
|
taxonomy = load_families()
|
|
cwes = {cwe for family in taxonomy.families if family.id in FAMILIES for cwe in family.cwes}
|
|
# family_of_cwe uses exact membership. If the taxonomy gains non-exact tokens,
|
|
# fetch broadly rather than risk excluding a client-selectable advisory.
|
|
# GitHub's advisories `cwes` filter expects bare numbers (89), not "CWE-89".
|
|
if cwes and all(re.fullmatch(r"CWE-[1-9][0-9]*", cwe) for cwe in cwes):
|
|
parameters["cwes"] = ",".join(sorted((cwe.removeprefix("CWE-") for cwe in cwes), key=int))
|
|
rows = self.get(next_url or "/advisories?" + urlencode(parameters))
|
|
if not isinstance(rows, list):
|
|
raise ValueError("invalid_advisory_page")
|
|
link = self.transport.response_headers.get("Link", "")
|
|
match = re.search(r'<([^>]+)>;\s*rel="next"', link)
|
|
return {"rows": rows, "next_url": match[1] if match else None}
|
|
|
|
def advisories(self):
|
|
"""Enumerate the candidate population, deduplicating multi-ecosystem advisories."""
|
|
seen = set()
|
|
for ecosystem in ECOSYSTEMS:
|
|
next_url = None
|
|
while True:
|
|
result = self.advisory_page(next_url, ecosystem=ecosystem)
|
|
for row in result["rows"]:
|
|
if row["ghsa_id"] not in seen:
|
|
seen.add(row["ghsa_id"])
|
|
yield row
|
|
next_url = result["next_url"]
|
|
if next_url is None:
|
|
break
|
|
|
|
|
|
class OSV:
|
|
def __init__(self, transport=None):
|
|
self.transport = transport or JSONTransport()
|
|
self.calls = 0
|
|
|
|
def get(self, identifier: str) -> dict:
|
|
if not re.fullmatch(r"GHSA-[\w-]+", identifier):
|
|
raise ValueError("invalid_advisory_id")
|
|
self.calls += 1
|
|
try:
|
|
return self.transport.get("https://api.osv.dev/v1/vulns/" + identifier, {})
|
|
except RepositoryNotFound:
|
|
return {} # No OSV bracket is distinct from a failed service call.
|