mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5.5
2b65680d5d
Sourced apart from the 12 pilot repositories: 212 candidates (pool-fresh, pool-retry, OSV for PyPI, npm, Maven and Packagist), 49 dropped because the benchmarks already name their repository, 413 catalog locations, 99 passing all three verifier checks in 28 repositories. Short of the 40-positive target. catalog.py now reads .java and .php. Freeze waits on the maintainer's spot-check (CANDIDATES.md, SOURCING.md). Co-Authored-By: Claude Opus 5.5 <[email protected]>
834 lines
38 KiB
Python
834 lines
38 KiB
Python
"""Sourcing run for the hunter-learning-loop TRAINING set (spec task 4.1, requirements R2.1-R2.3).
|
|
|
|
Subcommands, in the order they ran on 2026-10-13:
|
|
|
|
used the training-only used set (build_used minus measurements/populations; sweep without measurement text)
|
|
sources candidates: pool-fresh, pool-retry, then OSV (GHSA with one GitHub fix commit, permissive licence)
|
|
clone single-branch clones with history, fix + fix^1 fetched, parent read from git; disk guard
|
|
filter drop test, fixture, vendored and minified sites from the catalog
|
|
verify budgeted verifier run (counts calls, records provider errors by exception type only)
|
|
candidates CANDIDATES.md: every site that passed all three model checks, for the maintainer's spot-check
|
|
precheck ordinary-commit availability per repository with a passing site (what assemble will need)
|
|
|
|
Everything is deterministic given the seed and the OSV dump; network calls are counted and logged.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import tomllib
|
|
import zipfile
|
|
from collections import Counter, defaultdict
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
HERE = Path(__file__).resolve().parent
|
|
PILOT = ROOT / "benchmarks/measurements/2026-10-12-valid-benchmark-pilot"
|
|
POOL = ROOT / "benchmarks/unseen/pool-p1.toml"
|
|
HIST = Path.home() / "ousast-results/valid/train/hist"
|
|
# Intermediates, exclusion lists and per-batch files stay out of the tree: the repository guards in tests/ scan
|
|
# benchmarks/ for repository names, and an exclusion list is by construction a list of names used elsewhere.
|
|
WORK = Path.home() / "ousast-results/valid/train/work"
|
|
OSV_ECOSYSTEMS = {"PyPI": "pip", "npm": "npm", "Maven": "maven", "Packagist": "composer"}
|
|
MIN_FREE_BYTES = 6 * 1024**3
|
|
TEST_PATH = re.compile(
|
|
r"(^|/)(tests?|__tests__|spec|specs|fixtures?|testdata|e2e|__mocks__|mocks?)(/|$)|\.(test|spec)\.[cm]?[jt]sx?$|(^|/)test_[^/]*\.py$|_test\.py$|Test\.java$|Tests?\.php$",
|
|
re.I,
|
|
)
|
|
VENDORED_PATH = re.compile(r"(^|/)(node_modules|vendor|dist|build|third[_-]party|bower_components)(/|$)", re.I)
|
|
MINIFIED_PATH = re.compile(r"\.min\.[cm]?js$|\.bundle\.js$", re.I)
|
|
|
|
|
|
def log(message: str) -> None:
|
|
print(message, flush=True)
|
|
|
|
|
|
def free_bytes(path: Path) -> int:
|
|
usage = shutil.disk_usage(path)
|
|
return usage.free
|
|
|
|
|
|
def git(path: Path, *args: str, check: bool = True, timeout: int = 1800) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
["git", "-c", "credential.helper=", *args],
|
|
cwd=path,
|
|
stdin=subprocess.DEVNULL,
|
|
capture_output=True,
|
|
text=True,
|
|
check=check,
|
|
timeout=timeout,
|
|
env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
|
|
)
|
|
|
|
|
|
def pilot_accepted() -> set[str]:
|
|
return set(json.loads((PILOT / "accepted_repos.json").read_text()))
|
|
|
|
|
|
def pilot_sources() -> list[dict]:
|
|
return json.loads((PILOT / "sources.json").read_text())
|
|
|
|
|
|
def pool_public() -> list[dict]:
|
|
rows = tomllib.loads(POOL.read_text())["repository"]
|
|
return [row for row in rows if row.get("license_class") == "permissive"]
|
|
|
|
|
|
# ----------------------------------------------------------------------------- used
|
|
|
|
|
|
def cmd_used(args) -> int:
|
|
from benchmarks.unseen.eligibility import build_used, excluded, repositories
|
|
|
|
used = build_used(ROOT)
|
|
strict = set().union(*(names for label, names in used.sources.items() if label not in {"measurements", "populations"}))
|
|
# The sweep label reads benchmarks/measurements and the pilot's provenance as free text, which is the
|
|
# circularity NOTES.md names: a repository is not "used for training" because a measurement wrote its
|
|
# URL down. The refined set keeps every training label and sweeps only the tool's code, tests, plane
|
|
# and specs, not measurement records.
|
|
refined = set().union(*(names for label, names in used.sources.items() if label not in {"measurements", "populations", "sweep"}))
|
|
sweep_roots = [ROOT / top for top in ("src", "tests", "plane", ".kiro")] + [ROOT / "benchmarks"]
|
|
skip = (
|
|
ROOT / "benchmarks/unseen",
|
|
ROOT / "benchmarks/measurements",
|
|
ROOT / "benchmarks/experiments",
|
|
ROOT / "benchmarks/valid",
|
|
ROOT / "benchmarks/independent",
|
|
)
|
|
files = 0
|
|
for top in sweep_roots:
|
|
for path in top.rglob("*"):
|
|
if not path.is_file() or excluded(path) or any(path.is_relative_to(s) for s in skip):
|
|
continue
|
|
files += 1
|
|
refined.update(repositories(path.read_bytes().decode("utf-8", errors="ignore"), {}))
|
|
record = {
|
|
"labels": {label: len(names) for label, names in used.sources.items()},
|
|
"strict_count": len(strict),
|
|
"refined_count": len(refined),
|
|
"refined_sweep_files": files,
|
|
"refined_sweep_skips": [str(s.relative_to(ROOT)) for s in skip],
|
|
"pilot_repos_in_refined": sorted(pilot_accepted() & refined),
|
|
}
|
|
(WORK / "used_strict.json").write_text(json.dumps(sorted(strict), indent=0) + "\n")
|
|
(WORK / "used.json").write_text(json.dumps(sorted(refined), indent=0) + "\n")
|
|
(WORK / "used_log.json").write_text(json.dumps(record, indent=2, sort_keys=True) + "\n")
|
|
log(json.dumps(record, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
# ----------------------------------------------------------------------------- sources
|
|
|
|
|
|
def osv_text(record: dict) -> str:
|
|
summary = (record.get("summary") or "").strip()
|
|
details = (record.get("details") or "").strip()
|
|
return (summary + "\n\n" + details).strip() if summary else details
|
|
|
|
|
|
def cwe_for(record: dict, family: str, taxonomy) -> str | None:
|
|
for cwe in (record.get("database_specific") or {}).get("cwe_ids", []):
|
|
found = taxonomy.family_of_cwe(cwe)
|
|
if found and found.id == family:
|
|
return cwe
|
|
return None
|
|
|
|
|
|
def github_row(record: dict) -> dict:
|
|
"""An OSV GHSA record in the shape source.candidate expects (GitHub advisory API, so GitHub ecosystem names:
|
|
OSV's PyPI and Packagist become pip and composer)."""
|
|
specific = record.get("database_specific") or {}
|
|
return {
|
|
"ghsa_id": record["id"],
|
|
"published_at": record["published"],
|
|
"type": "reviewed" if specific.get("github_reviewed") else "unreviewed",
|
|
"withdrawn_at": record.get("withdrawn"),
|
|
"cwes": [{"cwe_id": c} for c in specific.get("cwe_ids", [])],
|
|
"vulnerabilities": [
|
|
{
|
|
"package": {
|
|
"ecosystem": OSV_ECOSYSTEMS.get(a["package"]["ecosystem"], a["package"]["ecosystem"]),
|
|
"name": a["package"]["name"],
|
|
}
|
|
}
|
|
for a in record.get("affected", [])
|
|
if a.get("package")
|
|
],
|
|
"references": [r.get("url", "") for r in record.get("references", [])],
|
|
}
|
|
|
|
|
|
class ZipOSV:
|
|
"""source.OSV look-alike that answers from the downloaded dump (no network)."""
|
|
|
|
def __init__(self, records: dict[str, dict]):
|
|
self.records = records
|
|
self.calls = 0
|
|
|
|
def get(self, identifier: str) -> dict:
|
|
self.calls += 1
|
|
return self.records.get(identifier, {})
|
|
|
|
|
|
def cmd_sources(args) -> int:
|
|
from benchmarks.unseen import source
|
|
from benchmarks.unseen.eligibility import RepositoryNotFound, normalize
|
|
from openultrasast.model.taxonomy import load_families
|
|
|
|
taxonomy = load_families()
|
|
accepted = pilot_accepted()
|
|
pilot = pilot_sources()
|
|
pilot_advisories = {row["advisory"] for row in pilot}
|
|
pilot_repos = {normalize(row["repository"]) for row in pilot}
|
|
public = pool_public()
|
|
pool_repos = {normalize(row["repository"]) for row in public}
|
|
used = set(json.loads((WORK / "used.json").read_text()))
|
|
excluded_repos = accepted | pilot_repos | pool_repos | used
|
|
log(
|
|
f"exclusion set: pilot accepted {len(accepted)}, pilot sourced {len(pilot_repos)}, pool public {len(pool_repos)}, used {len(used)} -> {len(excluded_repos)} names" # noqa: E501
|
|
)
|
|
|
|
osv_api = source.OSV()
|
|
rows: list[dict] = []
|
|
counts: Counter = Counter()
|
|
if args.extend:
|
|
# Second batch: only OSV, continuing the same seeded order past every repository already sourced.
|
|
earlier = json.loads((WORK / "sources.json").read_text())
|
|
excluded_repos |= {normalize(row["repository"]) for row in earlier}
|
|
log(f"extend: {len(earlier)} earlier sources excluded")
|
|
|
|
# (a) pool-fresh: permissive pool entries the pilot never sourced
|
|
for row in [] if args.extend else public:
|
|
repo = normalize(row["repository"])
|
|
if repo in accepted or row["advisory"] in pilot_advisories or repo in pilot_repos:
|
|
continue
|
|
record = osv_api.get(row["advisory"])
|
|
cwe = cwe_for(record, row["family"], taxonomy)
|
|
if not record or not cwe:
|
|
counts["pool-fresh:no_osv_record_or_cwe"] += 1
|
|
log(f"pool-fresh skip {row['advisory']}: no OSV record or no CWE in family {row['family']}")
|
|
continue
|
|
rows.append(
|
|
dict(
|
|
repository=repo,
|
|
repo=repo,
|
|
url=row["url"],
|
|
fix=row["fix"],
|
|
parent=row["parent"],
|
|
family=row["family"],
|
|
cwe=cwe,
|
|
advisory=row["advisory"],
|
|
advisory_text=osv_text(record),
|
|
ecosystem=row["ecosystem"],
|
|
license_spdx=row.get("license_spdx"),
|
|
origin="pool-fresh",
|
|
)
|
|
)
|
|
counts["pool-fresh"] += 1
|
|
|
|
# (b) pool-retry: the pilot sourced them, no site passed, repository not held out
|
|
for row in [] if args.extend else pilot:
|
|
repo = normalize(row["repository"])
|
|
if repo in accepted:
|
|
continue
|
|
rows.append({**row, "repository": repo, "repo": repo, "origin": "pool-retry"})
|
|
counts["pool-retry"] += 1
|
|
|
|
# (c) OSV dump: GHSA advisories with exactly one GitHub fix commit, one family, repo not excluded
|
|
pre: dict[str, dict] = {}
|
|
records: dict[str, dict] = {}
|
|
for eco_name, eco in OSV_ECOSYSTEMS.items():
|
|
archive = zipfile.ZipFile(Path(args.osv_dir) / f"{eco_name}.zip")
|
|
names = [n for n in archive.namelist() if n.startswith("GHSA-") and n.endswith(".json")]
|
|
counts[f"osv:{eco}:ghsa_records"] += len(names)
|
|
for name in names:
|
|
record = json.loads(archive.read(name))
|
|
row = github_row(record)
|
|
try:
|
|
if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1):
|
|
raise source.Rejected("date")
|
|
if row["type"] != "reviewed" or row.get("withdrawn_at"):
|
|
raise source.Rejected("advisory_status")
|
|
if source.ecosystem(row) != eco:
|
|
raise source.Rejected("ecosystem")
|
|
families = {f.id for c in row["cwes"] if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in source.FAMILIES}
|
|
if len(families) != 1:
|
|
raise source.Rejected("family")
|
|
links = source.fix_links(row)
|
|
if len(links) != 1:
|
|
raise source.Rejected("fix_links")
|
|
((repo, _),) = links
|
|
if repo in excluded_repos:
|
|
raise source.Rejected("excluded_repository")
|
|
except source.Rejected as exc:
|
|
counts[f"osv:{eco}:{exc}"] += 1
|
|
continue
|
|
counts[f"osv:{eco}:prefilter_pass"] += 1
|
|
records[record["id"]] = record
|
|
# one advisory per repository: the most recently published one
|
|
current = pre.get(repo)
|
|
if current is None or row["published_at"] > current["published_at"]:
|
|
pre[repo] = {**row, "_eco": eco}
|
|
log(f"OSV prefilter: {len(pre)} repositories with a candidate advisory")
|
|
counts["osv:prefilter_repositories"] = len(pre)
|
|
|
|
token = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True, check=True).stdout.strip()
|
|
github = source.GitHub(token)
|
|
osv_stub = ZipOSV(records)
|
|
ordered = source.ordered_advisories(list(pre.values()), args.seed)
|
|
target = args.target
|
|
picked = 0
|
|
for row in ordered:
|
|
if picked >= target:
|
|
break
|
|
try:
|
|
cand = source.candidate(row, github, osv_stub)
|
|
except source.Rejected as exc:
|
|
counts[f"osv:check:{exc}"] += 1
|
|
continue
|
|
except RepositoryNotFound:
|
|
counts["osv:check:not_found"] += 1
|
|
continue
|
|
except ValueError as exc:
|
|
counts[f"osv:check:{type(exc).__name__}:{exc}"] += 1
|
|
continue
|
|
if cand.license_class != "permissive":
|
|
counts["osv:check:licence_not_permissive"] += 1
|
|
continue
|
|
if cand.repository in excluded_repos:
|
|
counts["osv:check:excluded_after_resolution"] += 1
|
|
continue
|
|
record = records[row["ghsa_id"]]
|
|
cwe = cwe_for(record, cand.family, taxonomy)
|
|
rows.append(
|
|
dict(
|
|
repository=cand.repository,
|
|
repo=cand.repository,
|
|
url=cand.url,
|
|
fix=cand.fix,
|
|
parent=cand.parent,
|
|
family=cand.family,
|
|
cwe=cwe,
|
|
advisory=cand.advisory,
|
|
advisory_text=osv_text(record),
|
|
ecosystem=cand.ecosystem,
|
|
license_spdx=cand.license_spdx,
|
|
origin="osv-extend" if args.extend else "osv",
|
|
)
|
|
)
|
|
excluded_repos.add(cand.repository)
|
|
picked += 1
|
|
counts["osv"] += 1
|
|
log(f"osv pick {picked}/{target}: {cand.repository} {cand.advisory} {cand.ecosystem} {cand.family} {cand.license_spdx}")
|
|
counts["github_api_calls"] = github.calls
|
|
counts["osv_api_calls"] = osv_api.calls
|
|
# the candidate list must not contain a held-out repository, whatever the origin
|
|
assert not {row["repository"] for row in rows} & accepted
|
|
stem = "sources_extend" if args.extend else "sources"
|
|
(WORK / f"{stem}.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False) + "\n")
|
|
(WORK / f"{stem}_log.json").write_text(json.dumps(dict(sorted(counts.items())), indent=2) + "\n")
|
|
log(
|
|
json.dumps(
|
|
{k: v for k, v in counts.items() if not k.startswith("osv:") or "check" in k or "pass" in k or "repositories" in k},
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
log(f"sources.json: {len(rows)} advisories, {len({r['repository'] for r in rows})} repositories")
|
|
return 0
|
|
|
|
|
|
# ----------------------------------------------------------------------------- clone
|
|
|
|
|
|
def clone_one(repo: str, url: str, fix: str) -> tuple[Path | None, str, str | None]:
|
|
"""Clone single-branch with history, make sure fix and fix^1 are present, return (dir, status, parent)."""
|
|
target = HIST / repo.replace("/", "__")
|
|
if not (target / ".git").exists():
|
|
if target.exists():
|
|
shutil.rmtree(target)
|
|
done = subprocess.run(
|
|
["git", "-c", "credential.helper=", "clone", "--quiet", "--single-branch", "--no-tags", "--no-checkout", url, str(target)],
|
|
stdin=subprocess.DEVNULL,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=1800,
|
|
env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
|
|
)
|
|
if done.returncode:
|
|
shutil.rmtree(target, ignore_errors=True)
|
|
return None, "clone_failed", None
|
|
# No working tree: everything downstream reads objects (git show / log / diff). The first 64 clones were
|
|
# emptied the same way after the disk guard fired: `git sparse-checkout disable` restores a checkout.
|
|
git(target, "sparse-checkout", "set", "--no-cone", "/__worktree_emptied_for_disk__", check=False)
|
|
has_fix = git(target, "cat-file", "-e", f"{fix}^{{commit}}", check=False).returncode == 0
|
|
if not has_fix:
|
|
fetched = git(target, "fetch", "--quiet", "--depth", "2", "origin", fix, check=False)
|
|
if fetched.returncode:
|
|
fetched = git(target, "fetch", "--quiet", "origin", fix, check=False)
|
|
if fetched.returncode or git(target, "cat-file", "-e", f"{fix}^{{commit}}", check=False).returncode:
|
|
return target, "fix_unreachable", None
|
|
parents = git(target, "rev-list", "--parents", "-n", "1", fix).stdout.split()[1:]
|
|
if len(parents) != 1:
|
|
return target, f"parents_{len(parents)}", None
|
|
parent = parents[0]
|
|
if git(target, "cat-file", "-e", f"{parent}^{{commit}}", check=False).returncode:
|
|
fetched = git(target, "fetch", "--quiet", "--depth", "1", "origin", parent, check=False)
|
|
if fetched.returncode or git(target, "cat-file", "-e", f"{parent}^{{commit}}", check=False).returncode:
|
|
return target, "parent_unreachable", None
|
|
# prove the parent's tree is readable: the catalog reads blobs at fix^1
|
|
if git(target, "ls-tree", "-r", "--name-only", parent, check=False).returncode:
|
|
return target, "parent_tree_unreadable", None
|
|
return target, "ok", parent
|
|
|
|
|
|
def cmd_clone(args) -> int:
|
|
sources_path = Path(args.sources) if args.sources else WORK / "sources.json"
|
|
rows = json.loads(sources_path.read_text())
|
|
HIST.mkdir(parents=True, exist_ok=True)
|
|
repos_path = WORK / "repos_hist.json"
|
|
repos = json.loads(repos_path.read_text()) if repos_path.exists() else {}
|
|
clone_log_path = WORK / "clone_log.json"
|
|
clone_log = json.loads(clone_log_path.read_text()) if clone_log_path.exists() else {}
|
|
by_repo: dict[str, list[dict]] = defaultdict(list)
|
|
for row in rows:
|
|
by_repo[row["repository"]].append(row)
|
|
stopped = False
|
|
for index, (repo, entries) in enumerate(sorted(by_repo.items()), 1):
|
|
if repo in clone_log and clone_log[repo].get("status") == "ok" and all(e.get("parent_from_git") for e in entries):
|
|
continue
|
|
free = free_bytes(HIST)
|
|
if free < MIN_FREE_BYTES:
|
|
log(f"STOP: free space {free / 1024**3:.1f} GB under 6 GB before cloning {repo}")
|
|
stopped = True
|
|
break
|
|
started = time.monotonic()
|
|
url = entries[0]["url"]
|
|
status, parent_seen = "ok", {}
|
|
target = None
|
|
for entry in entries:
|
|
target, status, parent = clone_one(repo, url, entry["fix"])
|
|
parent_seen[entry["fix"]] = parent
|
|
if status != "ok":
|
|
break
|
|
size = sum(p.stat().st_size for p in target.rglob("*") if p.is_file()) if target and target.exists() else 0
|
|
commits = (
|
|
git(target, "rev-list", "--count", "--first-parent", "HEAD", check=False).stdout.strip() if target and status == "ok" else ""
|
|
)
|
|
clone_log[repo] = {
|
|
"status": status,
|
|
"bytes": size,
|
|
"seconds": round(time.monotonic() - started, 1),
|
|
"first_parent_commits": int(commits) if commits else None,
|
|
"parents": parent_seen,
|
|
}
|
|
if status == "ok":
|
|
repos[repo] = str(target)
|
|
for entry in entries:
|
|
entry["parent_from_git"] = parent_seen[entry["fix"]]
|
|
else:
|
|
if target and target.exists():
|
|
shutil.rmtree(target, ignore_errors=True)
|
|
repos.pop(repo, None)
|
|
log(
|
|
f"[{index}/{len(by_repo)}] {repo}: {status} {size / 1024**2:.1f} MB {clone_log[repo]['seconds']}s free={free_bytes(HIST) / 1024**3:.1f}GB" # noqa: E501
|
|
)
|
|
repos_path.write_text(json.dumps(dict(sorted(repos.items())), indent=1) + "\n")
|
|
clone_log_path.write_text(json.dumps(dict(sorted(clone_log.items())), indent=1) + "\n")
|
|
# rewrite sources: parent = fix^1 from git; move uncloned entries aside
|
|
kept, dropped = [], []
|
|
for row in rows:
|
|
entry = clone_log.get(row["repository"], {})
|
|
parent = row.get("parent_from_git") or entry.get("parents", {}).get(row["fix"])
|
|
if entry.get("status") == "ok" and parent:
|
|
if row["parent"] != parent:
|
|
row["source_parent"] = row["parent"]
|
|
row["parent"] = parent
|
|
row.pop("parent_from_git", None)
|
|
kept.append(row)
|
|
else:
|
|
dropped.append({**row, "clone_status": entry.get("status", "not_cloned")})
|
|
sources_path.write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n")
|
|
(sources_path.with_name(sources_path.stem + "_dropped.json")).write_text(json.dumps(dropped, indent=1, ensure_ascii=False) + "\n")
|
|
total = sum(e["bytes"] for e in clone_log.values())
|
|
log(
|
|
f"cloned {sum(e['status'] == 'ok' for e in clone_log.values())}/{len(by_repo)} repositories, {total / 1024**3:.2f} GB, kept {len(kept)} sources, dropped {len(dropped)}, stopped={stopped}" # noqa: E501
|
|
)
|
|
return 1 if stopped else 0
|
|
|
|
|
|
# ----------------------------------------------------------------------------- filter
|
|
|
|
|
|
def site_rejection(path: str) -> str | None:
|
|
if MINIFIED_PATH.search(path):
|
|
return "minified"
|
|
if VENDORED_PATH.search(path):
|
|
return "vendored"
|
|
if TEST_PATH.search(path):
|
|
return "test"
|
|
return None
|
|
|
|
|
|
def cmd_filter(args) -> int:
|
|
from benchmarks.valid.catalog import write_catalog
|
|
|
|
cases = tomllib.loads(Path(args.catalog).read_text())["case"]
|
|
kept, reasons = [], Counter()
|
|
for case in cases:
|
|
reason = site_rejection(case["path"])
|
|
if reason:
|
|
reasons[reason] += 1
|
|
continue
|
|
kept.append(case)
|
|
write_catalog(Path(args.out), kept)
|
|
log(f"catalog: {len(cases)} sites -> {len(kept)} kept, dropped {dict(reasons)}")
|
|
(WORK / "catalog_filter_log.json").write_text(
|
|
json.dumps({"input": len(cases), "kept": len(kept), "dropped": dict(reasons)}, indent=2) + "\n"
|
|
)
|
|
return 0
|
|
|
|
|
|
# ----------------------------------------------------------------------------- verify
|
|
|
|
|
|
class CountingClient:
|
|
def __init__(self, inner, budget: int):
|
|
self.inner, self.budget, self.calls = inner, budget, 0
|
|
|
|
def complete(self, **kwargs):
|
|
if self.calls >= self.budget:
|
|
raise BudgetExhausted()
|
|
self.calls += 1
|
|
return self.inner.complete(**kwargs)
|
|
|
|
|
|
class BudgetExhausted(RuntimeError):
|
|
pass
|
|
|
|
|
|
def cmd_verify(args) -> int:
|
|
from benchmarks.unseen.eligibility import normalize
|
|
from benchmarks.valid.catalog import local_git
|
|
from benchmarks.valid.verify import DEFAULT_DETECTOR_MODEL, DeepSeekChatClient, verify_case, write_records
|
|
from openultrasast.config import load_config
|
|
from openultrasast.model.endpoint import resolve_chat_endpoint
|
|
|
|
if not os.environ.get("DEEPSEEK_API_KEY"):
|
|
log("DEEPSEEK_API_KEY is not exported")
|
|
return 2
|
|
resolved = resolve_chat_endpoint(load_config(dotenv=False))
|
|
if resolved is None or resolved[1].provider != "deepseek":
|
|
log("DeepSeek provider is required")
|
|
return 2
|
|
client = resolved[0]
|
|
if isinstance(client, DeepSeekChatClient):
|
|
client = client.bounded(1200)
|
|
counting = CountingClient(client, args.budget)
|
|
repos = {normalize(k): local_git(Path(v)) for k, v in json.loads(Path(args.repos).read_text()).items()}
|
|
cases = tomllib.loads(Path(args.catalog).read_text())["case"]
|
|
out = Path(args.out)
|
|
records = {row["id"]: row for row in json.loads(out.read_text())["cases"]} if out.exists() else {}
|
|
log_path = WORK / "verify_log.json"
|
|
vlog = json.loads(log_path.read_text()) if log_path.exists() else {"calls": 0, "errors": {}, "passes": 0}
|
|
# round-robin across advisories so an advisory with many sites cannot eat the budget alone
|
|
groups: dict[str, list[dict]] = defaultdict(list)
|
|
for case in sorted(cases, key=lambda c: c["id"]):
|
|
groups[case["advisory"]].append(case)
|
|
order = []
|
|
depth = max(len(v) for v in groups.values())
|
|
for level in range(depth):
|
|
for advisory in sorted(groups):
|
|
if level < len(groups[advisory]):
|
|
order.append(groups[advisory][level])
|
|
errors: Counter = Counter(vlog.get("errors", {}))
|
|
started = time.monotonic()
|
|
done = 0
|
|
for case in order:
|
|
if case["id"] in records and not (args.retry_errors and records[case["id"]].get("error")):
|
|
continue
|
|
if counting.calls >= args.budget:
|
|
log(f"budget of {args.budget} calls reached")
|
|
break
|
|
try:
|
|
record = verify_case(repos[normalize(case["repo"])], case, client=counting)
|
|
except BudgetExhausted:
|
|
break
|
|
except Exception as exc: # provider or parse failure: record the type only, never the payload
|
|
record = {
|
|
"id": case["id"],
|
|
"verdict": "rejected",
|
|
"error": type(exc).__name__,
|
|
"model": DEFAULT_DETECTOR_MODEL,
|
|
"human_spot_check": None,
|
|
}
|
|
errors[type(exc).__name__] += 1
|
|
records[case["id"]] = record
|
|
done += 1
|
|
passed = record.get("verdict") == "pending_human"
|
|
log(
|
|
f"{case['id']} -> {record.get('verdict')}{' PASS' if passed else ''}{' error=' + record['error'] if record.get('error') else ''} calls={counting.calls}" # noqa: E501
|
|
)
|
|
if done % 5 == 0:
|
|
write_records(out, list(records.values()))
|
|
write_records(out, list(records.values()))
|
|
vlog["calls"] = vlog.get("calls", 0) + counting.calls
|
|
vlog["errors"] = dict(errors)
|
|
vlog["passes"] = sum(r.get("verdict") == "pending_human" for r in records.values())
|
|
vlog["records"] = len(records)
|
|
vlog["catalog_sites"] = len(cases)
|
|
vlog["seconds"] = vlog.get("seconds", 0) + round(time.monotonic() - started)
|
|
vlog["model"] = DEFAULT_DETECTOR_MODEL
|
|
vlog["budget"] = args.budget
|
|
log_path.write_text(json.dumps(vlog, indent=2, sort_keys=True) + "\n")
|
|
log(json.dumps(vlog, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
# ----------------------------------------------------------------------------- candidates
|
|
|
|
|
|
def cmd_candidates(args) -> int:
|
|
cases = {c["id"]: c for c in tomllib.loads(Path(args.catalog).read_text())["case"]}
|
|
records = json.loads(Path(args.verification).read_text())["cases"]
|
|
sources = {row["advisory"]: row for row in json.loads((HERE / "sources.json").read_text())}
|
|
passing = [r for r in records if r.get("verdict") == "pending_human"]
|
|
by_repo: dict[str, list[dict]] = defaultdict(list)
|
|
for record in passing:
|
|
by_repo[cases[record["id"]]["repo"]].append(record)
|
|
languages = Counter(cases[r["id"]]["language"] for r in passing)
|
|
changes = {(cases[r["id"]]["repo"], cases[r["id"]]["commit"]) for r in passing}
|
|
lines = [
|
|
"# Training-set candidates for the maintainer's spot-check",
|
|
"",
|
|
f"Generated {date.today().isoformat()} by `benchmarks/valid/train/sourcing.py candidates`. Every location below passed all three",
|
|
"model checks (vulnerable before the fix, fix addresses it, reachable from an in-repo entry point) and waits for a",
|
|
"human accept or reject. Nothing here is a positive until accepted. Accept or reject a location with",
|
|
"",
|
|
" .venv/bin/python -m benchmarks.valid.verify spot-check benchmarks/valid/train/verification.json <id> accept|reject --reviewer <you> --date YYYY-MM-DD", # noqa: E501
|
|
"",
|
|
f"Sites passing: {len(passing)} in {len(by_repo)} repositories, {len(changes)} positive changes (fix commits); languages: "
|
|
+ ", ".join(f"{k} {v}" for k, v in sorted(languages.items())),
|
|
"",
|
|
"The verifier's entry point is the model's claim, not a checked fact; it is the reason the site passed the reachability check.",
|
|
"",
|
|
]
|
|
for repo in sorted(by_repo):
|
|
rows = sorted(by_repo[repo], key=lambda r: r["id"])
|
|
first = cases[rows[0]["id"]]
|
|
src = sources.get(first["advisory"], {})
|
|
lines += [
|
|
f"## {repo}",
|
|
"",
|
|
f"- advisory: {first['advisory']} ({first['family']}, {first['cwe']}), origin `{src.get('origin', '?')}`, ecosystem {src.get('ecosystem', '?')}, licence {src.get('license_spdx', '?')}", # noqa: E501
|
|
f"- fix `{first['commit']}`, parent (fix^1) `{first['parent']}`",
|
|
f"- advisory title: {(first.get('advisory_text') or '').strip().splitlines()[0][:160] if first.get('advisory_text') else ''}",
|
|
"",
|
|
"| id | path::function (pre-fix lines) | language | entry point (verifier) | confidence |",
|
|
"|---|---|---|---|---|",
|
|
]
|
|
for record in rows:
|
|
case = cases[record["id"]]
|
|
span = case["function_lines"][0]
|
|
entry = str(record.get("entry_point", "")).replace("|", "\\|").replace("\n", " ")[:200]
|
|
lines.append(
|
|
f"| `{record['id']}` | `{case['path']}::{case['function']}` ({span[0]}-{span[1]}) | {case['language']} | {entry} | {record.get('confidence')} |" # noqa: E501
|
|
)
|
|
lines.append("")
|
|
Path(args.out).write_text("\n".join(lines) + "\n")
|
|
log(f"CANDIDATES.md: {len(passing)} sites, {len(by_repo)} repos, {len(changes)} changes, languages {dict(languages)}")
|
|
return 0
|
|
|
|
|
|
# ----------------------------------------------------------------------------- overlap
|
|
|
|
GITHUB = re.compile(r"github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", re.IGNORECASE)
|
|
REPO_LINE = re.compile(r'^\s*(?:repo|fix_repo)\s*=\s*"([^"]+)"', re.MULTILINE)
|
|
|
|
|
|
def named_elsewhere() -> dict[str, set[str]]:
|
|
"""Every repository the development tree already names, by where it is named. Mirrors the repository guards in
|
|
tests/ (test_advisory_fix_pairs, test_independent_population): github.com/<name> anywhere under benchmarks/, src/,
|
|
tests/, plane/ and .kiro/, plus `repo = "..."` lines, plus the strict used set. Vendored trees are skipped as the
|
|
pair guard skips them; this directory is skipped because it is what is being checked."""
|
|
from openultrasast.learn.labels import repo_name
|
|
|
|
found: dict[str, set[str]] = defaultdict(set)
|
|
for top in ("benchmarks", "src", "tests", "plane", ".kiro"):
|
|
for path in (ROOT / top).rglob("*"):
|
|
if "node_modules" in path.parts or "vendor" in path.parts or path.is_relative_to(HERE):
|
|
continue
|
|
if not path.is_file() or path.suffix in {".pyc", ".bin", ".gz"}:
|
|
continue
|
|
try:
|
|
text = path.read_text(errors="ignore")
|
|
except OSError:
|
|
continue
|
|
label = "benchmarks/" + path.relative_to(ROOT / "benchmarks").parts[0] if top == "benchmarks" else top
|
|
for match in GITHUB.findall(text):
|
|
found[repo_name(match)].add(label)
|
|
for match in REPO_LINE.findall(text):
|
|
with contextlib.suppress(Exception): # an unparseable value is not a repository name
|
|
found[repo_name(match)].add(label)
|
|
for name in json.loads((WORK / "used_strict.json").read_text()):
|
|
found[name].add("used_strict")
|
|
return found
|
|
|
|
|
|
def cmd_overlap(args) -> int:
|
|
"""Drop every candidate whose repository the tree already names; write the surviving sources to the tree."""
|
|
from openultrasast.learn.labels import repo_name
|
|
|
|
elsewhere = named_elsewhere()
|
|
rows = [row for name in args.batches for row in json.loads((WORK / name).read_text())]
|
|
kept, dropped = [], []
|
|
for row in rows:
|
|
where = sorted(elsewhere.get(repo_name(row["repository"]), ()))
|
|
if where:
|
|
dropped.append({"repository": row["repository"], "advisory": row["advisory"], "origin": row["origin"], "named_in": where})
|
|
else:
|
|
kept.append(row)
|
|
assert not {repo_name(r["repository"]) for r in kept} & pilot_accepted()
|
|
(WORK / "overlap_dropped.json").write_text(json.dumps(dropped, indent=1) + "\n")
|
|
(WORK / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n")
|
|
(HERE / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n")
|
|
log(
|
|
f"overlap: {len(rows)} sources, {len(dropped)} dropped (named elsewhere), {len(kept)} kept; by origin dropped {dict(Counter(d['origin'] for d in dropped))}" # noqa: E501
|
|
)
|
|
return 0
|
|
|
|
|
|
def cmd_merge(args) -> int:
|
|
"""Tree catalog.toml and verification.json: the batch files restricted to the surviving sources."""
|
|
from benchmarks.valid.catalog import write_catalog
|
|
from benchmarks.valid.verify import write_records
|
|
|
|
advisories = {row["advisory"] for row in json.loads((HERE / "sources.json").read_text())}
|
|
cases = [c for name in args.catalogs for c in tomllib.loads((WORK / name).read_text()).get("case", []) if c["advisory"] in advisories]
|
|
# A site whose code or advisory text links a repository the tree already names (a pair-corpus or reserved
|
|
# repository in a code comment, say) would put that name back into benchmarks/: drop the site, record why.
|
|
from openultrasast.learn.labels import repo_name
|
|
|
|
elsewhere = json.loads((WORK / "named_elsewhere.json").read_text())
|
|
dropped = []
|
|
for case in list(cases):
|
|
names = {repo_name(m) for value in case.values() if isinstance(value, str) for m in GITHUB.findall(value)} - {case["repo"]}
|
|
hits = sorted(n for n in names if n in elsewhere and not n.startswith("user-attachments/"))
|
|
if hits:
|
|
cases.remove(case)
|
|
dropped.append({"id": case["id"], "repo": case["repo"], "names": hits})
|
|
(WORK / "merge_dropped_sites.json").write_text(json.dumps(dropped, indent=1) + "\n")
|
|
ids = {c["id"] for c in cases}
|
|
records = [r for name in args.verifications for r in json.loads((WORK / name).read_text())["cases"] if r["id"] in ids]
|
|
write_catalog(HERE / "catalog.toml", cases)
|
|
write_records(HERE / "verification.json", records)
|
|
log(f"merge: {len(cases)} sites, {len(records)} verification records, {len(dropped)} sites dropped for naming a used repository")
|
|
return 0
|
|
|
|
|
|
# ----------------------------------------------------------------------------- precheck
|
|
|
|
|
|
def cmd_precheck(args) -> int:
|
|
from benchmarks.unseen import extract
|
|
from benchmarks.unseen.eligibility import normalize
|
|
from benchmarks.valid.catalog import local_git
|
|
|
|
cases = {c["id"]: c for c in tomllib.loads(Path(args.catalog).read_text())["case"]}
|
|
records = json.loads(Path(args.verification).read_text())["cases"]
|
|
repos = json.loads(Path(args.repos).read_text())
|
|
passing = defaultdict(list)
|
|
for record in records:
|
|
if record.get("verdict") == "pending_human":
|
|
passing[cases[record["id"]]["repo"]].append(cases[record["id"]])
|
|
result = {}
|
|
for repo in sorted(passing):
|
|
git_ = local_git(Path(repos[repo]))
|
|
all_cases = [c for c in cases.values() if normalize(c["repo"]) == repo]
|
|
fixes = {c["commit"] for c in all_cases}
|
|
functions = {(c["path"], c["function"]) for c in all_cases}
|
|
head = git_.run("rev-parse", "HEAD").strip()
|
|
started = time.monotonic()
|
|
try:
|
|
history = extract.ordinary_history(git_, head)
|
|
except Exception as exc: # assemble.freeze calls the same function and would stop on this repository
|
|
result[repo] = {"error": f"{type(exc).__name__}: {exc}", "seconds": round(time.monotonic() - started, 1)}
|
|
log(f"{repo}: ordinary_history failed: {type(exc).__name__}: {exc}")
|
|
(WORK / "precheck.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
|
continue
|
|
available = sum(extract.ordinary_rejection(row, fixes, functions) is None for row in history)
|
|
changes = len({(c["commit"], c["parent"]) for c in passing[repo]})
|
|
result[repo] = {
|
|
"first_parent_commits": len(history),
|
|
"ordinary_available": available,
|
|
"positive_changes": changes,
|
|
"needed_at_ratio_20": 20 * changes,
|
|
"enough": available >= 20 * changes,
|
|
"seconds": round(time.monotonic() - started, 1),
|
|
}
|
|
(WORK / "precheck.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
|
log(
|
|
f"{repo}: {available} ordinary of {len(history)} first-parent commits, needs {20 * changes}, enough={available >= 20 * changes}"
|
|
)
|
|
(WORK / "precheck.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
|
return 0
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
sub.add_parser("used")
|
|
p = sub.add_parser("sources")
|
|
p.add_argument("--osv-dir", required=True)
|
|
p.add_argument("--target", type=int, default=75, help="OSV advisories to accept after the GitHub checks")
|
|
p.add_argument("--seed", type=int, default=1)
|
|
p.add_argument("--extend", action="store_true", help="second OSV batch into sources_extend.json, excluding sources.json")
|
|
p = sub.add_parser("clone")
|
|
p.add_argument("--sources", help="sources file to clone (default sources.json)")
|
|
p = sub.add_parser("filter")
|
|
p.add_argument("catalog")
|
|
p.add_argument("--out", required=True)
|
|
p = sub.add_parser("verify")
|
|
p.add_argument("catalog")
|
|
p.add_argument("--repos", required=True)
|
|
p.add_argument("--out", required=True)
|
|
p.add_argument("--budget", type=int, default=600)
|
|
p.add_argument("--retry-errors", action="store_true")
|
|
p = sub.add_parser("candidates")
|
|
p.add_argument("verification")
|
|
p.add_argument("catalog")
|
|
p.add_argument("--out", required=True)
|
|
p = sub.add_parser("overlap")
|
|
p.add_argument("batches", nargs="+", help="source batch files in the work directory")
|
|
p = sub.add_parser("merge")
|
|
p.add_argument("--catalogs", nargs="+", required=True)
|
|
p.add_argument("--verifications", nargs="+", required=True)
|
|
p = sub.add_parser("precheck")
|
|
p.add_argument("verification")
|
|
p.add_argument("catalog")
|
|
p.add_argument("--repos", required=True)
|
|
args = parser.parse_args(argv)
|
|
return {
|
|
"used": cmd_used,
|
|
"sources": cmd_sources,
|
|
"clone": cmd_clone,
|
|
"filter": cmd_filter,
|
|
"verify": cmd_verify,
|
|
"candidates": cmd_candidates,
|
|
"precheck": cmd_precheck,
|
|
"overlap": cmd_overlap,
|
|
"merge": cmd_merge,
|
|
}[args.command](args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|