mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5.5
285bf58f78
The sourcing run was killed four times (exit 143/137) after growing to 4.6 GB RSS on the 7.7 GB VM. Live after the fix: flat 107 MB through 3,029 advisories. Implemented with Codex. Co-Authored-By: Claude Opus 5.5 <[email protected]>
306 lines
12 KiB
Python
306 lines
12 KiB
Python
"""Advisory selection. Network adapters are constructed only by the coordinator CLI."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import random
|
|
import re
|
|
from collections import OrderedDict
|
|
from dataclasses import dataclass
|
|
from datetime import date
|
|
from urllib.parse import urlencode, urlsplit
|
|
|
|
from openultrasast.model.taxonomy import load_families
|
|
|
|
from .eligibility import RepositoryNotFound, normalize
|
|
|
|
ECOSYSTEMS = ("pip", "npm", "maven", "composer")
|
|
OSV_ECOSYSTEMS = dict(zip(ECOSYSTEMS, ("PyPI", "npm", "Maven", "Packagist"), strict=True))
|
|
FAMILIES = {"injection", "path", "deserialization", "access_control", "output_encoding", "untrusted_destination", "config_secrets"}
|
|
PERMISSIVE = {"MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", "0BSD", "Unlicense", "Zlib", "PostgreSQL", "BSL-1.0"}
|
|
FIX = re.compile(r"https://github\.com/([\w.-]+/[\w.-]+)/commit/([a-fA-F0-9]{7,40})(?:[/?#].*)?$", re.I)
|
|
|
|
|
|
class Rejected(ValueError):
|
|
"""Expected eligibility rejection; messages are fixed reason codes, never names."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Candidate:
|
|
repository: str
|
|
url: str
|
|
fix: str
|
|
parent: str
|
|
advisory: str
|
|
ecosystem: str
|
|
family: str
|
|
post_cutoff: bool
|
|
first_affected: str | None
|
|
license_spdx: str
|
|
license_class: str
|
|
license_ref: str
|
|
license_path: str
|
|
|
|
|
|
def compact_advisory(row: dict) -> dict:
|
|
"""Keep exactly the input fields used by selection and the all-advisory fix index."""
|
|
return {
|
|
**{key: row[key] for key in ("ghsa_id", "published_at", "type", "withdrawn_at") if key in row},
|
|
"cwes": [{"cwe_id": c["cwe_id"]} for c in row.get("cwes", [])],
|
|
"references": [f"https://github.com/{name}/commit/{sha}" for name, sha in sorted(fix_links(row))],
|
|
"vulnerabilities": [{"package": {key: v["package"][key] for key in ("ecosystem", "name")}} for v in row.get("vulnerabilities", [])],
|
|
}
|
|
|
|
|
|
def compact_repository(row: dict) -> dict:
|
|
return {
|
|
**{key: row[key] for key in ("full_name", "private", "size", "clone_url", "former_names") if key in row},
|
|
**{key: {"full_name": row[key]["full_name"]} for key in ("parent", "source") if row.get(key)},
|
|
}
|
|
|
|
|
|
def post_cutoff(row: dict) -> bool:
|
|
return date.fromisoformat(row["published_at"][:10]) > date(2026, 6, 1)
|
|
|
|
|
|
def fix_links(row: dict) -> set[tuple[str, str]]:
|
|
links = set()
|
|
for reference in row.get("references", []):
|
|
url = reference if isinstance(reference, str) else reference.get("url", "")
|
|
match = FIX.fullmatch(url)
|
|
if match:
|
|
links.add((normalize(match[1]), match[2].lower()))
|
|
return links
|
|
|
|
|
|
def ecosystem(row: dict) -> str | None:
|
|
values = {v["package"]["ecosystem"].lower() for v in row.get("vulnerabilities", [])}
|
|
return next((e for e in ECOSYSTEMS if e in values), None)
|
|
|
|
|
|
def ordered_advisories(rows: list[dict], seed: int) -> list[dict]:
|
|
"""Stable seeded ordering, post-cutoff first in each ecosystem; round-robin strata."""
|
|
from .storage import AdvisoryRows
|
|
|
|
if isinstance(rows, AdvisoryRows):
|
|
return rows.ordered(seed)
|
|
rng = random.Random(seed)
|
|
groups = []
|
|
for eco in ECOSYSTEMS:
|
|
group = sorted((r for r in rows if ecosystem(r) == eco), key=lambda r: r["ghsa_id"])
|
|
rng.shuffle(group)
|
|
group.sort(key=lambda r: not post_cutoff(r))
|
|
groups.append(group)
|
|
return [group[i] for i in range(max(map(len, groups), default=0)) for group in groups if i < len(group)]
|
|
|
|
|
|
def first_affected(record: dict, row: dict, eco: str) -> str | None:
|
|
packages = {v["package"]["name"] for v in row["vulnerabilities"] if v["package"]["ecosystem"].lower() == eco}
|
|
versions = set()
|
|
for affected in record.get("affected", []):
|
|
package = affected.get("package", {})
|
|
if package.get("ecosystem") != OSV_ECOSYSTEMS[eco] or package.get("name") not in packages:
|
|
continue
|
|
for span in affected.get("ranges", []):
|
|
if span.get("type") not in {"ECOSYSTEM", "SEMVER"}:
|
|
continue
|
|
for event in span.get("events", []):
|
|
if event.get("introduced") not in {None, "0"}:
|
|
versions.add(event["introduced"])
|
|
if len(versions) > 1:
|
|
raise Rejected("ambiguous_version_bracket")
|
|
return next(iter(versions), None)
|
|
|
|
|
|
def license_info(record: dict) -> tuple[str, str]:
|
|
# Explicit pinned-file SPDX wins. For GitHub-recognized full texts its license endpoint
|
|
# provides the detected SPDX. Unknown/custom/compound declarations stay private.
|
|
declared = re.search(r"SPDX-License-Identifier:\s*([^\r\n]+)", record.get("text", ""))
|
|
spdx = declared[1].strip().removesuffix("*/").strip() if declared else record.get("spdx_id") or "NONE"
|
|
return spdx, "permissive" if spdx in PERMISSIVE else "private"
|
|
|
|
|
|
def candidate(row: dict, github, osv) -> Candidate:
|
|
if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1):
|
|
raise Rejected("date")
|
|
if row.get("type", "reviewed") != "reviewed" or row.get("withdrawn_at"):
|
|
raise Rejected("advisory_status")
|
|
eco = ecosystem(row)
|
|
if eco is None:
|
|
raise Rejected("ecosystem")
|
|
taxonomy = load_families()
|
|
families = {f.id for c in row.get("cwes", []) if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in FAMILIES}
|
|
if len(families) != 1:
|
|
raise Rejected("family")
|
|
links = fix_links(row)
|
|
if len(links) != 1:
|
|
raise Rejected("fix_links")
|
|
name, fix = next(iter(links))
|
|
repo = github.get_repo(name)
|
|
url = urlsplit(repo.get("clone_url", ""))
|
|
if (
|
|
repo.get("private", True)
|
|
or not 0 <= repo.get("size", -1) <= 500 * 1024
|
|
or url.scheme != "https"
|
|
or url.hostname != "github.com"
|
|
or url.username
|
|
or url.password
|
|
or url.query
|
|
or url.fragment
|
|
or normalize(repo["clone_url"]) != normalize(repo["full_name"])
|
|
):
|
|
raise Rejected("repository")
|
|
commit = github.get_commit(repo["full_name"], fix)
|
|
if len(commit["parents"]) != 1:
|
|
raise Rejected("parents")
|
|
parent = commit["parents"][0]["sha"]
|
|
license_record = github.get_license(repo["full_name"], parent)
|
|
spdx, license_class = license_info(license_record)
|
|
return Candidate(
|
|
normalize(repo["full_name"]),
|
|
repo["clone_url"],
|
|
commit["sha"],
|
|
parent,
|
|
row["ghsa_id"],
|
|
eco,
|
|
next(iter(families)),
|
|
post_cutoff(row),
|
|
first_affected(osv.get(row["ghsa_id"]), row, eco),
|
|
spdx,
|
|
license_class,
|
|
parent,
|
|
license_record.get("path", ""),
|
|
)
|
|
|
|
|
|
class JSONTransport:
|
|
"""Coordinator GET transport with bounded retries; all I/O and waits are injectable."""
|
|
|
|
def __init__(self, *, opener=None, sleeper=None, clock=None):
|
|
import time
|
|
from urllib.request import urlopen
|
|
|
|
self.opener = opener or urlopen
|
|
self.sleeper = sleeper or time.sleep
|
|
self.clock = clock or time.time
|
|
self.attempts = 0
|
|
self.rate_limit = {}
|
|
|
|
def get(self, url: str, headers: dict) -> object:
|
|
from urllib.error import HTTPError
|
|
from urllib.request import Request
|
|
|
|
for attempt in range(5):
|
|
self.attempts += 1
|
|
try:
|
|
with self.opener(Request(url, headers=headers), timeout=60) as response:
|
|
self.record_rate(getattr(response, "headers", {}))
|
|
return json.load(response)
|
|
except HTTPError as exc:
|
|
self.record_rate(exc.headers)
|
|
if exc.code == 404:
|
|
raise RepositoryNotFound("not_found") from None
|
|
limited = exc.code == 429 or (
|
|
exc.code == 403 and (exc.headers.get("X-RateLimit-Remaining") == "0" or exc.headers.get("Retry-After"))
|
|
)
|
|
transient = exc.code in {500, 502, 503, 504}
|
|
if attempt == 4 or not (limited or transient):
|
|
raise ValueError("http_failure") from None
|
|
try:
|
|
delay = float(exc.headers.get("Retry-After", "0"))
|
|
if limited and not delay:
|
|
delay = max(60, float(exc.headers.get("X-RateLimit-Reset", "0")) - self.clock() + 1)
|
|
delay = max(delay, 2**attempt)
|
|
if delay > 3605:
|
|
raise ValueError("retry_window")
|
|
except (TypeError, ValueError):
|
|
raise ValueError("http_retry_failed") from None
|
|
self.sleeper(delay)
|
|
except Exception:
|
|
raise ValueError("http_failure") from None
|
|
raise ValueError("http_failure")
|
|
|
|
def record_rate(self, headers):
|
|
self.rate_limit = {
|
|
key: headers.get(header)
|
|
for key, header in (
|
|
("remaining", "X-RateLimit-Remaining"),
|
|
("limit", "X-RateLimit-Limit"),
|
|
("reset", "X-RateLimit-Reset"),
|
|
("retry_after", "Retry-After"),
|
|
)
|
|
if headers.get(header) is not None
|
|
}
|
|
|
|
|
|
class GitHub:
|
|
def __init__(self, token: str, transport=None):
|
|
if not token:
|
|
raise ValueError("missing_gh_token")
|
|
self.transport = transport or JSONTransport()
|
|
self.headers = {"Authorization": "Bearer " + token, "Accept": "application/vnd.github+json"}
|
|
self.repos: OrderedDict[str, dict] = OrderedDict()
|
|
self.calls = 0
|
|
|
|
def get(self, path: str):
|
|
self.calls += 1
|
|
return self.transport.get("https://api.github.com" + path, self.headers)
|
|
|
|
def get_repo(self, name: str) -> dict:
|
|
name = normalize(name)
|
|
if name not in self.repos:
|
|
self.repos[name] = compact_repository(self.get("/repos/" + name))
|
|
if len(self.repos) > 128:
|
|
self.repos.popitem(last=False)
|
|
self.repos.move_to_end(name)
|
|
return self.repos[name]
|
|
|
|
def get_commit(self, name: str, sha: str) -> dict:
|
|
return self.get("/repos/" + normalize(name) + "/commits/" + sha)
|
|
|
|
def get_license(self, name: str, ref: str) -> dict:
|
|
try:
|
|
row = self.get("/repos/" + normalize(name) + "/license?" + urlencode({"ref": ref}))
|
|
except RepositoryNotFound:
|
|
return {"spdx_id": "NONE", "text": "", "path": ""}
|
|
return {
|
|
"spdx_id": (row.get("license") or {}).get("spdx_id"),
|
|
"path": row.get("path", ""),
|
|
"text": base64.b64decode(row.get("content", "")).decode("utf-8", errors="replace"),
|
|
}
|
|
|
|
def advisory_page(self, page: int) -> dict:
|
|
# Oldest first: newly published advisories do not shift already persisted pages.
|
|
rows = self.get(
|
|
"/advisories?" + urlencode({"type": "reviewed", "per_page": 100, "page": page, "sort": "published", "direction": "asc"})
|
|
)
|
|
if not isinstance(rows, list):
|
|
raise ValueError("invalid_advisory_page")
|
|
return {"rows": rows, "done": len(rows) < 100}
|
|
|
|
def advisories(self):
|
|
# Enumerate every reviewed advisory, including non-candidate fixes.
|
|
page = 1
|
|
while True:
|
|
result = self.advisory_page(page)
|
|
yield from result["rows"]
|
|
if result["done"]:
|
|
break
|
|
page += 1
|
|
|
|
|
|
class OSV:
|
|
def __init__(self, transport=None):
|
|
self.transport = transport or JSONTransport()
|
|
self.calls = 0
|
|
|
|
def get(self, identifier: str) -> dict:
|
|
if not re.fullmatch(r"GHSA-[\w-]+", identifier):
|
|
raise ValueError("invalid_advisory_id")
|
|
self.calls += 1
|
|
try:
|
|
return self.transport.get("https://api.osv.dev/v1/vulns/" + identifier, {})
|
|
except RepositoryNotFound:
|
|
return {} # No OSV bracket is distinct from a failed service call.
|