Open_Ultra_SAST/benchmarks/unseen/source.py

"""Advisory selection. Network adapters are constructed only by the coordinator CLI."""

from __future__ import annotations

import base64
import json
import random
import re
from dataclasses import dataclass
from datetime import date
from urllib.parse import urlencode, urlsplit

from openultrasast.model.taxonomy import load_families

from .eligibility import RepositoryNotFound, normalize

ECOSYSTEMS = ("pip", "npm", "maven", "composer")
OSV_ECOSYSTEMS = dict(zip(ECOSYSTEMS, ("PyPI", "npm", "Maven", "Packagist"), strict=True))
FAMILIES = {"injection", "path", "deserialization", "access_control", "output_encoding", "untrusted_destination", "config_secrets"}
PERMISSIVE = {"MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", "0BSD", "Unlicense", "Zlib", "PostgreSQL", "BSL-1.0"}
FIX = re.compile(r"https://github\.com/([\w.-]+/[\w.-]+)/commit/([a-fA-F0-9]{7,40})(?:[/?#].*)?$", re.I)


class Rejected(ValueError):
    """Expected eligibility rejection; messages are fixed reason codes, never names."""


@dataclass(frozen=True)
class Candidate:
    repository: str
    url: str
    fix: str
    parent: str
    advisory: str
    ecosystem: str
    family: str
    post_cutoff: bool
    first_affected: str | None
    license_spdx: str
    license_class: str
    license_ref: str
    license_path: str


def post_cutoff(row: dict) -> bool:
    return date.fromisoformat(row["published_at"][:10]) > date(2026, 6, 1)


def fix_links(row: dict) -> set[tuple[str, str]]:
    links = set()
    for reference in row.get("references", []):
        url = reference if isinstance(reference, str) else reference.get("url", "")
        match = FIX.fullmatch(url)
        if match:
            links.add((normalize(match[1]), match[2].lower()))
    return links


def ecosystem(row: dict) -> str | None:
    values = {v["package"]["ecosystem"].lower() for v in row.get("vulnerabilities", [])}
    return next((e for e in ECOSYSTEMS if e in values), None)


def reduce_advisory(row: dict) -> dict:
    """Keep only selection inputs, including every fix needed by the exclusion index.

    Preserve the parser's shape so fresh and resumed pages follow the same path.
    Commit parents, licenses and OSV brackets are resolved later into Candidate.
    """
    return {
        "ghsa_id": row["ghsa_id"],
        "published_at": row["published_at"],
        "type": row.get("type", "reviewed"),
        "withdrawn_at": row.get("withdrawn_at"),
        "cwes": [{"cwe_id": c["cwe_id"]} for c in row.get("cwes", [])],
        "vulnerabilities": [{"package": {key: v["package"][key] for key in ("ecosystem", "name")}} for v in row.get("vulnerabilities", [])],
        "references": [f"https://github.com/{name}/commit/{sha}" for name, sha in sorted(fix_links(row))],
    }


def ordered_advisories(rows: list[dict], seed: int) -> list[dict]:
    """Stable seeded ordering, post-cutoff first in each ecosystem; round-robin strata."""
    rng = random.Random(seed)
    groups = []
    for eco in ECOSYSTEMS:
        group = sorted((r for r in rows if ecosystem(r) == eco), key=lambda r: r["ghsa_id"])
        rng.shuffle(group)
        group.sort(key=lambda r: not post_cutoff(r))
        groups.append(group)
    return [group[i] for i in range(max(map(len, groups), default=0)) for group in groups if i < len(group)]


def first_affected(record: dict, row: dict, eco: str) -> str | None:
    packages = {v["package"]["name"] for v in row["vulnerabilities"] if v["package"]["ecosystem"].lower() == eco}
    versions = set()
    for affected in record.get("affected", []):
        package = affected.get("package", {})
        if package.get("ecosystem") != OSV_ECOSYSTEMS[eco] or package.get("name") not in packages:
            continue
        for span in affected.get("ranges", []):
            if span.get("type") not in {"ECOSYSTEM", "SEMVER"}:
                continue
            for event in span.get("events", []):
                if event.get("introduced") not in {None, "0"}:
                    versions.add(event["introduced"])
    if len(versions) > 1:
        raise Rejected("ambiguous_version_bracket")
    return next(iter(versions), None)


def license_info(record: dict) -> tuple[str, str]:
    # Explicit pinned-file SPDX wins. For GitHub-recognized full texts its license endpoint
    # provides the detected SPDX. Unknown/custom/compound declarations stay private.
    declared = re.search(r"SPDX-License-Identifier:\s*([^\r\n]+)", record.get("text", ""))
    spdx = declared[1].strip().removesuffix("*/").strip() if declared else record.get("spdx_id") or "NONE"
    return spdx, "permissive" if spdx in PERMISSIVE else "private"


def candidate(row: dict, github, osv) -> Candidate:
    if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1):
        raise Rejected("date")
    if row.get("type", "reviewed") != "reviewed" or row.get("withdrawn_at"):
        raise Rejected("advisory_status")
    eco = ecosystem(row)
    if eco is None:
        raise Rejected("ecosystem")
    taxonomy = load_families()
    families = {f.id for c in row.get("cwes", []) if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in FAMILIES}
    if len(families) != 1:
        raise Rejected("family")
    links = fix_links(row)
    if len(links) != 1:
        raise Rejected("fix_links")
    name, fix = next(iter(links))
    repo = github.get_repo(name)
    url = urlsplit(repo.get("clone_url", ""))
    if (
        repo.get("private", True)
        or not 0 <= repo.get("size", -1) <= 500 * 1024
        or url.scheme != "https"
        or url.hostname != "github.com"
        or url.username
        or url.password
        or url.query
        or url.fragment
        or normalize(repo["clone_url"]) != normalize(repo["full_name"])
    ):
        raise Rejected("repository")
    commit = github.get_commit(repo["full_name"], fix)
    if len(commit["parents"]) != 1:
        raise Rejected("parents")
    parent = commit["parents"][0]["sha"]
    license_record = github.get_license(repo["full_name"], parent)
    spdx, license_class = license_info(license_record)
    return Candidate(
        normalize(repo["full_name"]),
        repo["clone_url"],
        commit["sha"],
        parent,
        row["ghsa_id"],
        eco,
        next(iter(families)),
        post_cutoff(row),
        first_affected(osv.get(row["ghsa_id"]), row, eco),
        spdx,
        license_class,
        parent,
        license_record.get("path", ""),
    )


class JSONTransport:
    """Coordinator GET transport with bounded retries; all I/O and waits are injectable."""

    def __init__(self, *, opener=None, sleeper=None, clock=None):
        import time
        from urllib.request import urlopen

        self.opener = opener or urlopen
        self.sleeper = sleeper or time.sleep
        self.clock = clock or time.time
        self.attempts = 0
        self.rate_limit = {}

    def get(self, url: str, headers: dict) -> object:
        from http.client import IncompleteRead
        from urllib.error import HTTPError, URLError
        from urllib.request import Request

        for attempt in range(5):
            self.attempts += 1
            retry_headers, limited = {}, False
            try:
                with self.opener(Request(url, headers=headers), timeout=60) as response:
                    retry_headers = getattr(response, "headers", {})
                    self.record_rate(retry_headers)
                    return json.load(response)
            except HTTPError as exc:
                self.record_rate(exc.headers)
                if exc.code == 404:
                    raise RepositoryNotFound("not_found") from None
                limited = exc.code == 429 or (
                    exc.code == 403 and (exc.headers.get("X-RateLimit-Remaining") == "0" or exc.headers.get("Retry-After"))
                )
                transient = exc.code in {500, 502, 503, 504}
                if attempt == 4 or not (limited or transient):
                    raise ValueError("http_failure") from None
                retry_headers = exc.headers
            except (URLError, TimeoutError, ConnectionError, IncompleteRead, json.JSONDecodeError):
                if attempt == 4:
                    raise ValueError("http_failure") from None
            except Exception:
                raise ValueError("http_failure") from None
            try:
                delay = float(retry_headers.get("Retry-After", "0"))
                if limited and not delay:
                    delay = max(60, float(retry_headers.get("X-RateLimit-Reset", "0")) - self.clock() + 1)
                delay = max(delay, 2**attempt)
                if delay > 3605:
                    raise ValueError("retry_window")
            except (TypeError, ValueError):
                raise ValueError("http_retry_failed") from None
            self.sleeper(delay)
        raise ValueError("http_failure")

    def record_rate(self, headers):
        self.rate_limit = {
            key: headers.get(header)
            for key, header in (
                ("remaining", "X-RateLimit-Remaining"),
                ("limit", "X-RateLimit-Limit"),
                ("reset", "X-RateLimit-Reset"),
                ("retry_after", "Retry-After"),
            )
            if headers.get(header) is not None
        }


class GitHub:
    def __init__(self, token: str, transport=None):
        if not token:
            raise ValueError("missing_gh_token")
        self.transport = transport or JSONTransport()
        self.headers = {"Authorization": "Bearer " + token, "Accept": "application/vnd.github+json"}
        self.repos: dict[str, dict] = {}
        self.calls = 0

    def get(self, path: str):
        self.calls += 1
        return self.transport.get("https://api.github.com" + path, self.headers)

    def get_repo(self, name: str) -> dict:
        name = normalize(name)
        if name not in self.repos:
            self.repos[name] = self.get("/repos/" + name)
        return self.repos[name]

    def get_commit(self, name: str, sha: str) -> dict:
        return self.get("/repos/" + normalize(name) + "/commits/" + sha)

    def get_license(self, name: str, ref: str) -> dict:
        try:
            row = self.get("/repos/" + normalize(name) + "/license?" + urlencode({"ref": ref}))
        except RepositoryNotFound:
            return {"spdx_id": "NONE", "text": "", "path": ""}
        return {
            "spdx_id": (row.get("license") or {}).get("spdx_id"),
            "path": row.get("path", ""),
            "text": base64.b64decode(row.get("content", "")).decode("utf-8", errors="replace"),
        }

    def advisory_page(self, page: int) -> dict:
        # Oldest first: newly published advisories do not shift already persisted pages.
        rows = self.get(
            "/advisories?" + urlencode({"type": "reviewed", "per_page": 100, "page": page, "sort": "published", "direction": "asc"})
        )
        if not isinstance(rows, list):
            raise ValueError("invalid_advisory_page")
        return {"rows": rows, "done": len(rows) < 100}

    def advisories(self):
        # Enumerate every reviewed advisory, including non-candidate fixes.
        page = 1
        while True:
            result = self.advisory_page(page)
            yield from result["rows"]
            if result["done"]:
                break
            page += 1


class OSV:
    def __init__(self, transport=None):
        self.transport = transport or JSONTransport()
        self.calls = 0

    def get(self, identifier: str) -> dict:
        if not re.fullmatch(r"GHSA-[\w-]+", identifier):
            raise ValueError("invalid_advisory_id")
        self.calls += 1
        try:
            return self.transport.get("https://api.osv.dev/v1/vulns/" + identifier, {})
        except RepositoryNotFound:
            return {}  # No OSV bracket is distinct from a failed service call.