"""Sourcing run for the hunter-learning-loop TRAINING set (spec task 4.1, requirements R2.1-R2.3). Subcommands, in the order they ran on 2026-10-13: used the training-only used set (build_used minus measurements/populations; sweep without measurement text) sources candidates: pool-fresh, pool-retry, then OSV (GHSA with one GitHub fix commit, permissive licence) clone single-branch clones with history, fix + fix^1 fetched, parent read from git; disk guard filter drop test, fixture, vendored and minified sites from the catalog verify budgeted verifier run (counts calls, records provider errors by exception type only) candidates CANDIDATES.md: every site that passed all three model checks, for the maintainer's spot-check precheck ordinary-commit availability per repository with a passing site (what assemble will need) Everything is deterministic given the seed and the OSV dump; network calls are counted and logged. """ from __future__ import annotations import argparse import contextlib import json import os import re import shutil import subprocess import sys import time import tomllib import zipfile from collections import Counter, defaultdict from datetime import date from pathlib import Path ROOT = Path(__file__).resolve().parents[3] HERE = Path(__file__).resolve().parent PILOT = ROOT / "benchmarks/measurements/2026-10-12-valid-benchmark-pilot" POOL = ROOT / "benchmarks/unseen/pool-p1.toml" HIST = Path.home() / "ousast-results/valid/train/hist" # Intermediates, exclusion lists and per-batch files stay out of the tree: the repository guards in tests/ scan # benchmarks/ for repository names, and an exclusion list is by construction a list of names used elsewhere. WORK = Path.home() / "ousast-results/valid/train/work" OSV_ECOSYSTEMS = {"PyPI": "pip", "npm": "npm", "Maven": "maven", "Packagist": "composer"} MIN_FREE_BYTES = 6 * 1024**3 TEST_PATH = re.compile( r"(^|/)(tests?|__tests__|spec|specs|fixtures?|testdata|e2e|__mocks__|mocks?)(/|$)|\.(test|spec)\.[cm]?[jt]sx?$|(^|/)test_[^/]*\.py$|_test\.py$|Test\.java$|Tests?\.php$", re.I, ) VENDORED_PATH = re.compile(r"(^|/)(node_modules|vendor|dist|build|third[_-]party|bower_components)(/|$)", re.I) MINIFIED_PATH = re.compile(r"\.min\.[cm]?js$|\.bundle\.js$", re.I) def log(message: str) -> None: print(message, flush=True) def free_bytes(path: Path) -> int: usage = shutil.disk_usage(path) return usage.free def git(path: Path, *args: str, check: bool = True, timeout: int = 1800) -> subprocess.CompletedProcess: return subprocess.run( ["git", "-c", "credential.helper=", *args], cwd=path, stdin=subprocess.DEVNULL, capture_output=True, text=True, check=check, timeout=timeout, env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}, ) def pilot_accepted() -> set[str]: return set(json.loads((PILOT / "accepted_repos.json").read_text())) def pilot_sources() -> list[dict]: return json.loads((PILOT / "sources.json").read_text()) def pool_public() -> list[dict]: rows = tomllib.loads(POOL.read_text())["repository"] return [row for row in rows if row.get("license_class") == "permissive"] # ----------------------------------------------------------------------------- used def cmd_used(args) -> int: from benchmarks.unseen.eligibility import build_used, excluded, repositories used = build_used(ROOT) strict = set().union(*(names for label, names in used.sources.items() if label not in {"measurements", "populations"})) # The sweep label reads benchmarks/measurements and the pilot's provenance as free text, which is the # circularity NOTES.md names: a repository is not "used for training" because a measurement wrote its # URL down. The refined set keeps every training label and sweeps only the tool's code, tests, plane # and specs, not measurement records. refined = set().union(*(names for label, names in used.sources.items() if label not in {"measurements", "populations", "sweep"})) sweep_roots = [ROOT / top for top in ("src", "tests", "plane", ".kiro")] + [ROOT / "benchmarks"] skip = ( ROOT / "benchmarks/unseen", ROOT / "benchmarks/measurements", ROOT / "benchmarks/experiments", ROOT / "benchmarks/valid", ROOT / "benchmarks/independent", ) files = 0 for top in sweep_roots: for path in top.rglob("*"): if not path.is_file() or excluded(path) or any(path.is_relative_to(s) for s in skip): continue files += 1 refined.update(repositories(path.read_bytes().decode("utf-8", errors="ignore"), {})) record = { "labels": {label: len(names) for label, names in used.sources.items()}, "strict_count": len(strict), "refined_count": len(refined), "refined_sweep_files": files, "refined_sweep_skips": [str(s.relative_to(ROOT)) for s in skip], "pilot_repos_in_refined": sorted(pilot_accepted() & refined), } (WORK / "used_strict.json").write_text(json.dumps(sorted(strict), indent=0) + "\n") (WORK / "used.json").write_text(json.dumps(sorted(refined), indent=0) + "\n") (WORK / "used_log.json").write_text(json.dumps(record, indent=2, sort_keys=True) + "\n") log(json.dumps(record, sort_keys=True)) return 0 # ----------------------------------------------------------------------------- sources def osv_text(record: dict) -> str: summary = (record.get("summary") or "").strip() details = (record.get("details") or "").strip() return (summary + "\n\n" + details).strip() if summary else details def cwe_for(record: dict, family: str, taxonomy) -> str | None: for cwe in (record.get("database_specific") or {}).get("cwe_ids", []): found = taxonomy.family_of_cwe(cwe) if found and found.id == family: return cwe return None def github_row(record: dict) -> dict: """An OSV GHSA record in the shape source.candidate expects (GitHub advisory API, so GitHub ecosystem names: OSV's PyPI and Packagist become pip and composer).""" specific = record.get("database_specific") or {} return { "ghsa_id": record["id"], "published_at": record["published"], "type": "reviewed" if specific.get("github_reviewed") else "unreviewed", "withdrawn_at": record.get("withdrawn"), "cwes": [{"cwe_id": c} for c in specific.get("cwe_ids", [])], "vulnerabilities": [ { "package": { "ecosystem": OSV_ECOSYSTEMS.get(a["package"]["ecosystem"], a["package"]["ecosystem"]), "name": a["package"]["name"], } } for a in record.get("affected", []) if a.get("package") ], "references": [r.get("url", "") for r in record.get("references", [])], } class ZipOSV: """source.OSV look-alike that answers from the downloaded dump (no network).""" def __init__(self, records: dict[str, dict]): self.records = records self.calls = 0 def get(self, identifier: str) -> dict: self.calls += 1 return self.records.get(identifier, {}) def family_exclusions() -> dict[str, set[str]]: """Collect strict, pilot, tree and earlier batch exclusions before GitHub checks.""" from benchmarks.unseen.eligibility import normalize def normalized(names) -> set[str]: out = set() for name in names: try: out.add(normalize(name)) except ValueError: continue # a link fragment that is not a repository name excludes nothing return out groups = { "pilot": normalized(pilot_accepted()), "named_elsewhere": normalized(named_elsewhere()), "used_strict": normalized(json.loads((WORK / "used_strict.json").read_text())) if (WORK / "used_strict.json").is_file() else set(), "already_sourced": set(), } for directory in (WORK, HERE): paths = {directory / "sources.json", *directory.glob("sources_batch*.json")} for path in sorted(paths): if not path.is_file() or path.stem.endswith("_log"): continue groups["already_sourced"].update(normalize(row["repository"]) for row in json.loads(path.read_text())) return groups def cmd_sources(args) -> int: from benchmarks.unseen import source from benchmarks.unseen.eligibility import RepositoryNotFound, normalize from openultrasast.model.taxonomy import load_families taxonomy = load_families() family = args.family if family: log(f"family {family}: {', '.join(sorted(taxonomy.by_id(family).cwes))}") accepted = {normalize(name) for name in pilot_accepted()} pilot = [] if family else pilot_sources() pilot_advisories = {row["advisory"] for row in pilot} pilot_repos = {normalize(row["repository"]) for row in pilot} public = [] if family else pool_public() pool_repos = {normalize(row["repository"]) for row in public} used = set() if family else set(json.loads((WORK / "used.json").read_text())) excluded_repos = accepted | pilot_repos | pool_repos | used exclusion_groups = family_exclusions() if family else {"repository": excluded_repos} excluded_repos |= set().union(*exclusion_groups.values()) if family: log(f"exclusion set: { {reason: len(names) for reason, names in exclusion_groups.items()} } -> {len(excluded_repos)} names") else: log( f"exclusion set: pilot accepted {len(accepted)}, pilot sourced {len(pilot_repos)}, pool public {len(pool_repos)}, used {len(used)} -> {len(excluded_repos)} names" # noqa: E501 ) osv_api = source.OSV() rows: list[dict] = [] counts: Counter = Counter() if args.extend and not family: # Second batch: only OSV, continuing the same seeded order past every repository already sourced. earlier = json.loads((WORK / "sources.json").read_text()) excluded_repos |= {normalize(row["repository"]) for row in earlier} log(f"extend: {len(earlier)} earlier sources excluded") # (a) pool-fresh: permissive pool entries the pilot never sourced for row in [] if args.extend else public: repo = normalize(row["repository"]) if repo in accepted or row["advisory"] in pilot_advisories or repo in pilot_repos: continue record = osv_api.get(row["advisory"]) cwe = cwe_for(record, row["family"], taxonomy) if not record or not cwe: counts["pool-fresh:no_osv_record_or_cwe"] += 1 log(f"pool-fresh skip {row['advisory']}: no OSV record or no CWE in family {row['family']}") continue rows.append( dict( repository=repo, repo=repo, url=row["url"], fix=row["fix"], parent=row["parent"], family=row["family"], cwe=cwe, advisory=row["advisory"], advisory_text=osv_text(record), ecosystem=row["ecosystem"], license_spdx=row.get("license_spdx"), origin="pool-fresh", ) ) counts["pool-fresh"] += 1 # (b) pool-retry: the pilot sourced them, no site passed, repository not held out for row in [] if args.extend else pilot: repo = normalize(row["repository"]) if repo in accepted: continue rows.append({**row, "repository": repo, "repo": repo, "origin": "pool-retry"}) counts["pool-retry"] += 1 # (c) OSV dump: GHSA advisories with exactly one GitHub fix commit, one family, repo not excluded pre: dict[str, dict] = {} records: dict[str, dict] = {} for eco_name, eco in OSV_ECOSYSTEMS.items(): archive_path = Path(args.osv_dir) / f"{eco_name}.zip" counts[f"osv:{eco}:ghsa_records"] = 0 counts[f"osv:{eco}:prefilter_pass"] = 0 for record in dump_records(archive_path): counts[f"osv:{eco}:ghsa_records"] += 1 row = github_row(record) try: if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1): raise source.Rejected("date") if row["type"] != "reviewed" or row.get("withdrawn_at"): raise source.Rejected("advisory_status") if source.ecosystem(row) != eco: raise source.Rejected("ecosystem") families = {f.id for c in row["cwes"] if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in source.FAMILIES} if len(families) != 1: raise source.Rejected("family") if family and families != {family}: raise source.Rejected("family_mismatch") links = source.fix_links(row) if len(links) != 1: raise source.Rejected("fix_links") ((repo, _),) = links if repo in excluded_repos: reason = next(reason for reason, names in exclusion_groups.items() if repo in names) raise source.Rejected(f"excluded_{reason}") except source.Rejected as exc: counts[f"osv:{eco}:{exc}"] += 1 continue counts[f"osv:{eco}:prefilter_pass"] += 1 records[record["id"]] = record # one advisory per repository: the most recently published one current = pre.get(repo) if current is None or row["published_at"] > current["published_at"]: pre[repo] = {**row, "_eco": eco} log(json.dumps({k: v for k, v in sorted(counts.items()) if k.startswith(f"osv:{eco}:")})) log(f"OSV prefilter: {len(pre)} repositories with a candidate advisory") counts["osv:prefilter_repositories"] = len(pre) github = None if not args.prefilter_only: token = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True, check=True).stdout.strip() github = source.GitHub(token) else: log("PREFILTER ONLY: GitHub checks did not run; no advisories are accepted.") osv_stub = ZipOSV(records) ordered = source.ordered_advisories(list(pre.values()), args.seed) target = args.target if args.target is not None else (120 if family else 75) counts["target"] = target counts["github_checks_run"] = int(github is not None) counts["accepted"] = 0 picked = 0 for row in ordered if github is not None else []: if picked >= target: break try: cand = source.candidate(row, github, osv_stub) except source.Rejected as exc: counts[f"osv:check:{exc}"] += 1 continue except RepositoryNotFound: counts["osv:check:not_found"] += 1 continue except ValueError as exc: counts[f"osv:check:{type(exc).__name__}:{exc}"] += 1 continue if cand.license_class != "permissive": counts["osv:check:licence_not_permissive"] += 1 continue if cand.repository in excluded_repos: counts["osv:check:excluded_after_resolution"] += 1 continue record = records[row["ghsa_id"]] cwe = cwe_for(record, cand.family, taxonomy) rows.append( dict( repository=cand.repository, repo=cand.repository, url=cand.url, fix=cand.fix, parent=cand.parent, family=cand.family, cwe=cwe, advisory=cand.advisory, advisory_text=osv_text(record), ecosystem=cand.ecosystem, license_spdx=cand.license_spdx, origin="osv-extend" if args.extend else "osv", ) ) excluded_repos.add(cand.repository) picked += 1 counts["osv"] += 1 counts["accepted"] += 1 log(f"osv pick {picked}/{target}: {cand.repository} {cand.advisory} {cand.ecosystem} {cand.family} {cand.license_spdx}") counts["github_api_calls"] = github.calls if github is not None else 0 counts["osv_api_calls"] = osv_api.calls # the candidate list must not contain a held-out repository, whatever the origin assert not {row["repository"] for row in rows} & accepted stem = f"sources_{family}" if family else ("sources_extend" if args.extend else "sources") output = HERE if family else WORK (output / f"{stem}.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False) + "\n") (output / f"{stem}_log.json").write_text(json.dumps(dict(sorted(counts.items())), indent=2) + "\n") log(json.dumps(dict(sorted(counts.items())))) log(f"{stem}.json: {len(rows)} advisories, {len({r['repository'] for r in rows})} repositories") return 0 def dump_records(path: Path): """Read GHSA records from a dump and report the bytes read from the first record.""" with zipfile.ZipFile(path) as archive: first = True for name in archive.namelist(): if not name.startswith("GHSA-") or not name.endswith(".json"): continue data = archive.read(name) if first: log(f"read {path}: {name}, {len(data)} bytes") first = False yield json.loads(data) # ----------------------------------------------------------------------------- clone def clone_one(repo: str, url: str, fix: str) -> tuple[Path | None, str, str | None]: """Clone single-branch with history, make sure fix and fix^1 are present, return (dir, status, parent).""" target = HIST / repo.replace("/", "__") if not (target / ".git").exists(): if target.exists(): shutil.rmtree(target) done = subprocess.run( ["git", "-c", "credential.helper=", "clone", "--quiet", "--single-branch", "--no-tags", "--no-checkout", url, str(target)], stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=1800, env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}, ) if done.returncode: shutil.rmtree(target, ignore_errors=True) return None, "clone_failed", None # No working tree: everything downstream reads objects (git show / log / diff). The first 64 clones were # emptied the same way after the disk guard fired: `git sparse-checkout disable` restores a checkout. git(target, "sparse-checkout", "set", "--no-cone", "/__worktree_emptied_for_disk__", check=False) has_fix = git(target, "cat-file", "-e", f"{fix}^{{commit}}", check=False).returncode == 0 if not has_fix: fetched = git(target, "fetch", "--quiet", "--depth", "2", "origin", fix, check=False) if fetched.returncode: fetched = git(target, "fetch", "--quiet", "origin", fix, check=False) if fetched.returncode or git(target, "cat-file", "-e", f"{fix}^{{commit}}", check=False).returncode: return target, "fix_unreachable", None parents = git(target, "rev-list", "--parents", "-n", "1", fix).stdout.split()[1:] if len(parents) != 1: return target, f"parents_{len(parents)}", None parent = parents[0] if git(target, "cat-file", "-e", f"{parent}^{{commit}}", check=False).returncode: fetched = git(target, "fetch", "--quiet", "--depth", "1", "origin", parent, check=False) if fetched.returncode or git(target, "cat-file", "-e", f"{parent}^{{commit}}", check=False).returncode: return target, "parent_unreachable", None # prove the parent's tree is readable: the catalog reads blobs at fix^1 if git(target, "ls-tree", "-r", "--name-only", parent, check=False).returncode: return target, "parent_tree_unreadable", None return target, "ok", parent def cmd_clone(args) -> int: sources_path = Path(args.sources) if args.sources else WORK / "sources.json" rows = json.loads(sources_path.read_text()) HIST.mkdir(parents=True, exist_ok=True) repos_path = WORK / "repos_hist.json" repos = json.loads(repos_path.read_text()) if repos_path.exists() else {} clone_log_path = WORK / "clone_log.json" clone_log = json.loads(clone_log_path.read_text()) if clone_log_path.exists() else {} by_repo: dict[str, list[dict]] = defaultdict(list) for row in rows: by_repo[row["repository"]].append(row) stopped = False for index, (repo, entries) in enumerate(sorted(by_repo.items()), 1): if repo in clone_log and clone_log[repo].get("status") == "ok" and all(e.get("parent_from_git") for e in entries): continue free = free_bytes(HIST) if free < MIN_FREE_BYTES: log(f"STOP: free space {free / 1024**3:.1f} GB under 6 GB before cloning {repo}") stopped = True break started = time.monotonic() url = entries[0]["url"] status, parent_seen = "ok", {} target = None for entry in entries: target, status, parent = clone_one(repo, url, entry["fix"]) parent_seen[entry["fix"]] = parent if status != "ok": break size = sum(p.stat().st_size for p in target.rglob("*") if p.is_file()) if target and target.exists() else 0 commits = ( git(target, "rev-list", "--count", "--first-parent", "HEAD", check=False).stdout.strip() if target and status == "ok" else "" ) clone_log[repo] = { "status": status, "bytes": size, "seconds": round(time.monotonic() - started, 1), "first_parent_commits": int(commits) if commits else None, "parents": parent_seen, } if status == "ok": repos[repo] = str(target) for entry in entries: entry["parent_from_git"] = parent_seen[entry["fix"]] else: if target and target.exists(): shutil.rmtree(target, ignore_errors=True) repos.pop(repo, None) log( f"[{index}/{len(by_repo)}] {repo}: {status} {size / 1024**2:.1f} MB {clone_log[repo]['seconds']}s free={free_bytes(HIST) / 1024**3:.1f}GB" # noqa: E501 ) repos_path.write_text(json.dumps(dict(sorted(repos.items())), indent=1) + "\n") clone_log_path.write_text(json.dumps(dict(sorted(clone_log.items())), indent=1) + "\n") # rewrite sources: parent = fix^1 from git; move uncloned entries aside kept, dropped = [], [] for row in rows: entry = clone_log.get(row["repository"], {}) parent = row.get("parent_from_git") or entry.get("parents", {}).get(row["fix"]) if entry.get("status") == "ok" and parent: if row["parent"] != parent: row["source_parent"] = row["parent"] row["parent"] = parent row.pop("parent_from_git", None) kept.append(row) else: dropped.append({**row, "clone_status": entry.get("status", "not_cloned")}) sources_path.write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n") (sources_path.with_name(sources_path.stem + "_dropped.json")).write_text(json.dumps(dropped, indent=1, ensure_ascii=False) + "\n") total = sum(e["bytes"] for e in clone_log.values()) log( f"cloned {sum(e['status'] == 'ok' for e in clone_log.values())}/{len(by_repo)} repositories, {total / 1024**3:.2f} GB, kept {len(kept)} sources, dropped {len(dropped)}, stopped={stopped}" # noqa: E501 ) return 1 if stopped else 0 # ----------------------------------------------------------------------------- filter def site_rejection(path: str) -> str | None: if MINIFIED_PATH.search(path): return "minified" if VENDORED_PATH.search(path): return "vendored" if TEST_PATH.search(path): return "test" return None def cmd_filter(args) -> int: from benchmarks.valid.catalog import write_catalog cases = tomllib.loads(Path(args.catalog).read_text())["case"] kept, reasons = [], Counter() for case in cases: reason = site_rejection(case["path"]) if reason: reasons[reason] += 1 continue kept.append(case) write_catalog(Path(args.out), kept) log(f"catalog: {len(cases)} sites -> {len(kept)} kept, dropped {dict(reasons)}") (WORK / "catalog_filter_log.json").write_text( json.dumps({"input": len(cases), "kept": len(kept), "dropped": dict(reasons)}, indent=2) + "\n" ) return 0 # ----------------------------------------------------------------------------- verify class CountingClient: def __init__(self, inner, budget: int): self.inner, self.budget, self.calls = inner, budget, 0 def complete(self, **kwargs): if self.calls >= self.budget: raise BudgetExhausted() self.calls += 1 return self.inner.complete(**kwargs) class BudgetExhausted(RuntimeError): pass def cmd_verify(args) -> int: from benchmarks.unseen.eligibility import normalize from benchmarks.valid.catalog import local_git from benchmarks.valid.verify import DEFAULT_DETECTOR_MODEL, DeepSeekChatClient, verify_case, write_records from openultrasast.config import load_config from openultrasast.model.endpoint import resolve_chat_endpoint if not os.environ.get("DEEPSEEK_API_KEY"): log("DEEPSEEK_API_KEY is not exported") return 2 resolved = resolve_chat_endpoint(load_config(dotenv=False)) if resolved is None or resolved[1].provider != "deepseek": log("DeepSeek provider is required") return 2 client = resolved[0] if isinstance(client, DeepSeekChatClient): client = client.bounded(1200) counting = CountingClient(client, args.budget) repos = {normalize(k): local_git(Path(v)) for k, v in json.loads(Path(args.repos).read_text()).items()} cases = tomllib.loads(Path(args.catalog).read_text())["case"] out = Path(args.out) records = {row["id"]: row for row in json.loads(out.read_text())["cases"]} if out.exists() else {} log_path = WORK / "verify_log.json" vlog = json.loads(log_path.read_text()) if log_path.exists() else {"calls": 0, "errors": {}, "passes": 0} # round-robin across advisories so an advisory with many sites cannot eat the budget alone groups: dict[str, list[dict]] = defaultdict(list) for case in sorted(cases, key=lambda c: c["id"]): groups[case["advisory"]].append(case) order = [] depth = max(len(v) for v in groups.values()) for level in range(depth): for advisory in sorted(groups): if level < len(groups[advisory]): order.append(groups[advisory][level]) errors: Counter = Counter(vlog.get("errors", {})) started = time.monotonic() done = 0 for case in order: if case["id"] in records and not (args.retry_errors and records[case["id"]].get("error")): continue if counting.calls >= args.budget: log(f"budget of {args.budget} calls reached") break try: record = verify_case(repos[normalize(case["repo"])], case, client=counting) except BudgetExhausted: break except Exception as exc: # provider or parse failure: record the type only, never the payload record = { "id": case["id"], "verdict": "rejected", "error": type(exc).__name__, "model": DEFAULT_DETECTOR_MODEL, "human_spot_check": None, } errors[type(exc).__name__] += 1 records[case["id"]] = record done += 1 passed = record.get("verdict") == "pending_human" log( f"{case['id']} -> {record.get('verdict')}{' PASS' if passed else ''}{' error=' + record['error'] if record.get('error') else ''} calls={counting.calls}" # noqa: E501 ) if done % 5 == 0: write_records(out, list(records.values())) write_records(out, list(records.values())) vlog["calls"] = vlog.get("calls", 0) + counting.calls vlog["errors"] = dict(errors) vlog["passes"] = sum(r.get("verdict") == "pending_human" for r in records.values()) vlog["records"] = len(records) vlog["catalog_sites"] = len(cases) vlog["seconds"] = vlog.get("seconds", 0) + round(time.monotonic() - started) vlog["model"] = DEFAULT_DETECTOR_MODEL vlog["budget"] = args.budget log_path.write_text(json.dumps(vlog, indent=2, sort_keys=True) + "\n") log(json.dumps(vlog, sort_keys=True)) return 0 # ----------------------------------------------------------------------------- candidates def cmd_candidates(args) -> int: cases = {c["id"]: c for c in tomllib.loads(Path(args.catalog).read_text())["case"]} records = json.loads(Path(args.verification).read_text())["cases"] sources = {row["advisory"]: row for row in json.loads((HERE / "sources.json").read_text())} passing = [r for r in records if r.get("verdict") == "pending_human"] by_repo: dict[str, list[dict]] = defaultdict(list) for record in passing: by_repo[cases[record["id"]]["repo"]].append(record) languages = Counter(cases[r["id"]]["language"] for r in passing) changes = {(cases[r["id"]]["repo"], cases[r["id"]]["commit"]) for r in passing} lines = [ "# Training-set candidates for the maintainer's spot-check", "", f"Generated {date.today().isoformat()} by `benchmarks/valid/train/sourcing.py candidates`. Every location below passed all three", "model checks (vulnerable before the fix, fix addresses it, reachable from an in-repo entry point) and waits for a", "human accept or reject. Nothing here is a positive until accepted. Accept or reject a location with", "", " .venv/bin/python -m benchmarks.valid.verify spot-check benchmarks/valid/train/verification.json accept|reject --reviewer --date YYYY-MM-DD", # noqa: E501 "", f"Sites passing: {len(passing)} in {len(by_repo)} repositories, {len(changes)} positive changes (fix commits); languages: " + ", ".join(f"{k} {v}" for k, v in sorted(languages.items())), "", "The verifier's entry point is the model's claim, not a checked fact; it is the reason the site passed the reachability check.", "", ] for repo in sorted(by_repo): rows = sorted(by_repo[repo], key=lambda r: r["id"]) first = cases[rows[0]["id"]] src = sources.get(first["advisory"], {}) lines += [ f"## {repo}", "", f"- advisory: {first['advisory']} ({first['family']}, {first['cwe']}), origin `{src.get('origin', '?')}`, ecosystem {src.get('ecosystem', '?')}, licence {src.get('license_spdx', '?')}", # noqa: E501 f"- fix `{first['commit']}`, parent (fix^1) `{first['parent']}`", f"- advisory title: {(first.get('advisory_text') or '').strip().splitlines()[0][:160] if first.get('advisory_text') else ''}", "", "| id | path::function (pre-fix lines) | language | entry point (verifier) | confidence |", "|---|---|---|---|---|", ] for record in rows: case = cases[record["id"]] span = case["function_lines"][0] entry = str(record.get("entry_point", "")).replace("|", "\\|").replace("\n", " ")[:200] lines.append( f"| `{record['id']}` | `{case['path']}::{case['function']}` ({span[0]}-{span[1]}) | {case['language']} | {entry} | {record.get('confidence')} |" # noqa: E501 ) lines.append("") Path(args.out).write_text("\n".join(lines) + "\n") log(f"CANDIDATES.md: {len(passing)} sites, {len(by_repo)} repos, {len(changes)} changes, languages {dict(languages)}") return 0 # ----------------------------------------------------------------------------- overlap GITHUB = re.compile(r"github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", re.IGNORECASE) REPO_LINE = re.compile(r'^\s*(?:repo|fix_repo)\s*=\s*"([^"]+)"', re.MULTILINE) def named_elsewhere() -> dict[str, set[str]]: """Every repository the development tree already names, by where it is named. Mirrors the repository guards in tests/ (test_advisory_fix_pairs, test_independent_population): github.com/ anywhere under benchmarks/, src/, tests/, plane/ and .kiro/, plus `repo = "..."` lines, plus the strict used set. Vendored trees are skipped as the pair guard skips them; this directory is skipped because it is what is being checked.""" from openultrasast.learn.labels import repo_name found: dict[str, set[str]] = defaultdict(set) for top in ("benchmarks", "src", "tests", "plane", ".kiro"): for path in (ROOT / top).rglob("*"): if "node_modules" in path.parts or "vendor" in path.parts or path.is_relative_to(HERE): continue if not path.is_file() or path.suffix in {".pyc", ".bin", ".gz"}: continue try: text = path.read_text(errors="ignore") except OSError: continue label = "benchmarks/" + path.relative_to(ROOT / "benchmarks").parts[0] if top == "benchmarks" else top for match in GITHUB.findall(text): found[repo_name(match)].add(label) for match in REPO_LINE.findall(text): with contextlib.suppress(Exception): # an unparseable value is not a repository name found[repo_name(match)].add(label) for name in json.loads((WORK / "used_strict.json").read_text()): found[name].add("used_strict") return found def cmd_overlap(args) -> int: """Drop every candidate whose repository the tree already names; write the surviving sources to the tree.""" from openultrasast.learn.labels import repo_name elsewhere = named_elsewhere() if args.family: for reason, names in family_exclusions().items(): for name in names: elsewhere.setdefault(name, set()).add(reason) rows = [row for name in args.batches for row in json.loads((WORK / name).read_text())] kept, dropped = [], [] for row in rows: if args.family and row["family"] != args.family: continue where = sorted(elsewhere.get(repo_name(row["repository"]), ())) if where: dropped.append({"repository": row["repository"], "advisory": row["advisory"], "origin": row["origin"], "named_in": where}) else: kept.append(row) assert not {repo_name(r["repository"]) for r in kept} & pilot_accepted() suffix = f"_{args.family}" if args.family else "" (WORK / f"overlap{suffix}_dropped.json").write_text(json.dumps(dropped, indent=1) + "\n") if not args.family: (WORK / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n") (HERE / f"sources{suffix}.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n") log( f"overlap: {len(rows)} sources, {len(dropped)} dropped (named elsewhere), {len(kept)} kept; by origin dropped {dict(Counter(d['origin'] for d in dropped))}" # noqa: E501 ) return 0 def cmd_merge(args) -> int: """Tree catalog.toml and verification.json: the batch files restricted to the surviving sources.""" from benchmarks.valid.catalog import write_catalog from benchmarks.valid.verify import write_records suffix = f"_{args.family}" if args.family else "" advisories = {row["advisory"] for row in json.loads((HERE / f"sources{suffix}.json").read_text())} cases = [c for name in args.catalogs for c in tomllib.loads((WORK / name).read_text()).get("case", []) if c["advisory"] in advisories] # A site whose code or advisory text links a repository the tree already names (a pair-corpus or reserved # repository in a code comment, say) would put that name back into benchmarks/: drop the site, record why. from openultrasast.learn.labels import repo_name elsewhere = json.loads((WORK / "named_elsewhere.json").read_text()) dropped = [] for case in list(cases): names = {repo_name(m) for value in case.values() if isinstance(value, str) for m in GITHUB.findall(value)} - {case["repo"]} hits = sorted(n for n in names if n in elsewhere and not n.startswith("user-attachments/")) if hits: cases.remove(case) dropped.append({"id": case["id"], "repo": case["repo"], "names": hits}) (WORK / f"merge{suffix}_dropped_sites.json").write_text(json.dumps(dropped, indent=1) + "\n") ids = {c["id"] for c in cases} records = [r for name in args.verifications for r in json.loads((WORK / name).read_text())["cases"] if r["id"] in ids] write_catalog(HERE / f"catalog{suffix}.toml", cases) write_records(HERE / f"verification{suffix}.json", records) log(f"merge: {len(cases)} sites, {len(records)} verification records, {len(dropped)} sites dropped for naming a used repository") return 0 # ----------------------------------------------------------------------------- precheck def cmd_precheck(args) -> int: from benchmarks.unseen import extract from benchmarks.unseen.eligibility import normalize from benchmarks.valid.catalog import local_git cases = {c["id"]: c for c in tomllib.loads(Path(args.catalog).read_text())["case"]} records = json.loads(Path(args.verification).read_text())["cases"] repos = json.loads(Path(args.repos).read_text()) passing = defaultdict(list) for record in records: if record.get("verdict") == "pending_human": passing[cases[record["id"]]["repo"]].append(cases[record["id"]]) result = {} for repo in sorted(passing): git_ = local_git(Path(repos[repo])) all_cases = [c for c in cases.values() if normalize(c["repo"]) == repo] fixes = {c["commit"] for c in all_cases} functions = {(c["path"], c["function"]) for c in all_cases} head = git_.run("rev-parse", "HEAD").strip() started = time.monotonic() try: history = extract.ordinary_history(git_, head) except Exception as exc: # assemble.freeze calls the same function and would stop on this repository result[repo] = {"error": f"{type(exc).__name__}: {exc}", "seconds": round(time.monotonic() - started, 1)} log(f"{repo}: ordinary_history failed: {type(exc).__name__}: {exc}") (WORK / "precheck.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") continue available = sum(extract.ordinary_rejection(row, fixes, functions) is None for row in history) changes = len({(c["commit"], c["parent"]) for c in passing[repo]}) result[repo] = { "first_parent_commits": len(history), "ordinary_available": available, "positive_changes": changes, "needed_at_ratio_20": 20 * changes, "enough": available >= 20 * changes, "seconds": round(time.monotonic() - started, 1), } (WORK / "precheck.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") log( f"{repo}: {available} ordinary of {len(history)} first-parent commits, needs {20 * changes}, enough={available >= 20 * changes}" ) (WORK / "precheck.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") return 0 def main(argv=None) -> int: from benchmarks.unseen.source import FAMILIES parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("used") p = sub.add_parser("sources") p.add_argument("--osv-dir", required=True) p.add_argument("--target", type=int, help="OSV advisories to accept after GitHub checks (default: 120 with --family, otherwise 75)") p.add_argument("--seed", type=int, default=1) mode = p.add_mutually_exclusive_group() mode.add_argument("--extend", action="store_true", help="second OSV batch into sources_extend.json, excluding sources.json") mode.add_argument("--family", choices=sorted(FAMILIES), help="source one family into sources_.json in the training directory") p.add_argument( "--prefilter-only", action="store_true", help="with --family, read dumps and exclusions without GitHub checks or accepts" ) p = sub.add_parser("clone") p.add_argument("--sources", help="sources file to clone (default sources.json)") p = sub.add_parser("filter") p.add_argument("catalog") p.add_argument("--out", required=True) p = sub.add_parser("verify") p.add_argument("catalog") p.add_argument("--repos", required=True) p.add_argument("--out", required=True) p.add_argument("--budget", type=int, default=600) p.add_argument("--retry-errors", action="store_true") p = sub.add_parser("candidates") p.add_argument("verification") p.add_argument("catalog") p.add_argument("--out", required=True) p = sub.add_parser("overlap") p.add_argument("--family", choices=sorted(FAMILIES)) p.add_argument("batches", nargs="+", help="source batch files in the work directory") p = sub.add_parser("merge") p.add_argument("--family", choices=sorted(FAMILIES)) p.add_argument("--catalogs", nargs="+", required=True) p.add_argument("--verifications", nargs="+", required=True) p = sub.add_parser("precheck") p.add_argument("verification") p.add_argument("catalog") p.add_argument("--repos", required=True) args = parser.parse_args(argv) if args.command == "sources" and args.prefilter_only and not args.family: parser.error("--prefilter-only requires --family") return { "used": cmd_used, "sources": cmd_sources, "clone": cmd_clone, "filter": cmd_filter, "verify": cmd_verify, "candidates": cmd_candidates, "precheck": cmd_precheck, "overlap": cmd_overlap, "merge": cmd_merge, }[args.command](args) if __name__ == "__main__": sys.exit(main())