mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
834 lines
38 KiB
Python
834 lines
38 KiB
Python
|
"""Sourcing run for the hunter-learning-loop TRAINING set (spec task 4.1, requirements R2.1-R2.3).
|
||
|
|
|
||
|
|
Subcommands, in the order they ran on 2026-10-13:
|
||
|
|
|
||
|
|
used the training-only used set (build_used minus measurements/populations; sweep without measurement text)
|
||
|
|
sources candidates: pool-fresh, pool-retry, then OSV (GHSA with one GitHub fix commit, permissive licence)
|
||
|
|
clone single-branch clones with history, fix + fix^1 fetched, parent read from git; disk guard
|
||
|
|
filter drop test, fixture, vendored and minified sites from the catalog
|
||
|
|
verify budgeted verifier run (counts calls, records provider errors by exception type only)
|
||
|
|
candidates CANDIDATES.md: every site that passed all three model checks, for the maintainer's spot-check
|
||
|
|
precheck ordinary-commit availability per repository with a passing site (what assemble will need)
|
||
|
|
|
||
|
|
Everything is deterministic given the seed and the OSV dump; network calls are counted and logged.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import contextlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import shutil
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
import tomllib
|
||
|
|
import zipfile
|
||
|
|
from collections import Counter, defaultdict
|
||
|
|
from datetime import date
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
ROOT = Path(__file__).resolve().parents[3]
|
||
|
|
HERE = Path(__file__).resolve().parent
|
||
|
|
PILOT = ROOT / "benchmarks/measurements/2026-10-12-valid-benchmark-pilot"
|
||
|
|
POOL = ROOT / "benchmarks/unseen/pool-p1.toml"
|
||
|
|
HIST = Path.home() / "ousast-results/valid/train/hist"
|
||
|
|
# Intermediates, exclusion lists and per-batch files stay out of the tree: the repository guards in tests/ scan
|
||
|
|
# benchmarks/ for repository names, and an exclusion list is by construction a list of names used elsewhere.
|
||
|
|
WORK = Path.home() / "ousast-results/valid/train/work"
|
||
|
|
OSV_ECOSYSTEMS = {"PyPI": "pip", "npm": "npm", "Maven": "maven", "Packagist": "composer"}
|
||
|
|
MIN_FREE_BYTES = 6 * 1024**3
|
||
|
|
TEST_PATH = re.compile(
|
||
|
|
r"(^|/)(tests?|__tests__|spec|specs|fixtures?|testdata|e2e|__mocks__|mocks?)(/|$)|\.(test|spec)\.[cm]?[jt]sx?$|(^|/)test_[^/]*\.py$|_test\.py$|Test\.java$|Tests?\.php$",
|
||
|
|
re.I,
|
||
|
|
)
|
||
|
|
VENDORED_PATH = re.compile(r"(^|/)(node_modules|vendor|dist|build|third[_-]party|bower_components)(/|$)", re.I)
|
||
|
|
MINIFIED_PATH = re.compile(r"\.min\.[cm]?js$|\.bundle\.js$", re.I)
|
||
|
|
|
||
|
|
|
||
|
|
def log(message: str) -> None:
|
||
|
|
print(message, flush=True)
|
||
|
|
|
||
|
|
|
||
|
|
def free_bytes(path: Path) -> int:
|
||
|
|
usage = shutil.disk_usage(path)
|
||
|
|
return usage.free
|
||
|
|
|
||
|
|
|
||
|
|
def git(path: Path, *args: str, check: bool = True, timeout: int = 1800) -> subprocess.CompletedProcess:
|
||
|
|
return subprocess.run(
|
||
|
|
["git", "-c", "credential.helper=", *args],
|
||
|
|
cwd=path,
|
||
|
|
stdin=subprocess.DEVNULL,
|
||
|
|
capture_output=True,
|
||
|
|
text=True,
|
||
|
|
check=check,
|
||
|
|
timeout=timeout,
|
||
|
|
env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def pilot_accepted() -> set[str]:
|
||
|
|
return set(json.loads((PILOT / "accepted_repos.json").read_text()))
|
||
|
|
|
||
|
|
|
||
|
|
def pilot_sources() -> list[dict]:
|
||
|
|
return json.loads((PILOT / "sources.json").read_text())
|
||
|
|
|
||
|
|
|
||
|
|
def pool_public() -> list[dict]:
|
||
|
|
rows = tomllib.loads(POOL.read_text())["repository"]
|
||
|
|
return [row for row in rows if row.get("license_class") == "permissive"]
|
||
|
|
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------- used
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_used(args) -> int:
|
||
|
|
from benchmarks.unseen.eligibility import build_used, excluded, repositories
|
||
|
|
|
||
|
|
used = build_used(ROOT)
|
||
|
|
strict = set().union(*(names for label, names in used.sources.items() if label not in {"measurements", "populations"}))
|
||
|
|
# The sweep label reads benchmarks/measurements and the pilot's provenance as free text, which is the
|
||
|
|
# circularity NOTES.md names: a repository is not "used for training" because a measurement wrote its
|
||
|
|
# URL down. The refined set keeps every training label and sweeps only the tool's code, tests, plane
|
||
|
|
# and specs, not measurement records.
|
||
|
|
refined = set().union(*(names for label, names in used.sources.items() if label not in {"measurements", "populations", "sweep"}))
|
||
|
|
sweep_roots = [ROOT / top for top in ("src", "tests", "plane", ".kiro")] + [ROOT / "benchmarks"]
|
||
|
|
skip = (
|
||
|
|
ROOT / "benchmarks/unseen",
|
||
|
|
ROOT / "benchmarks/measurements",
|
||
|
|
ROOT / "benchmarks/experiments",
|
||
|
|
ROOT / "benchmarks/valid",
|
||
|
|
ROOT / "benchmarks/independent",
|
||
|
|
)
|
||
|
|
files = 0
|
||
|
|
for top in sweep_roots:
|
||
|
|
for path in top.rglob("*"):
|
||
|
|
if not path.is_file() or excluded(path) or any(path.is_relative_to(s) for s in skip):
|
||
|
|
continue
|
||
|
|
files += 1
|
||
|
|
refined.update(repositories(path.read_bytes().decode("utf-8", errors="ignore"), {}))
|
||
|
|
record = {
|
||
|
|
"labels": {label: len(names) for label, names in used.sources.items()},
|
||
|
|
"strict_count": len(strict),
|
||
|
|
"refined_count": len(refined),
|
||
|
|
"refined_sweep_files": files,
|
||
|
|
"refined_sweep_skips": [str(s.relative_to(ROOT)) for s in skip],
|
||
|
|
"pilot_repos_in_refined": sorted(pilot_accepted() & refined),
|
||
|
|
}
|
||
|
|
(WORK / "used_strict.json").write_text(json.dumps(sorted(strict), indent=0) + "\n")
|
||
|
|
(WORK / "used.json").write_text(json.dumps(sorted(refined), indent=0) + "\n")
|
||
|
|
(WORK / "used_log.json").write_text(json.dumps(record, indent=2, sort_keys=True) + "\n")
|
||
|
|
log(json.dumps(record, sort_keys=True))
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------- sources
|
||
|
|
|
||
|
|
|
||
|
|
def osv_text(record: dict) -> str:
|
||
|
|
summary = (record.get("summary") or "").strip()
|
||
|
|
details = (record.get("details") or "").strip()
|
||
|
|
return (summary + "\n\n" + details).strip() if summary else details
|
||
|
|
|
||
|
|
|
||
|
|
def cwe_for(record: dict, family: str, taxonomy) -> str | None:
|
||
|
|
for cwe in (record.get("database_specific") or {}).get("cwe_ids", []):
|
||
|
|
found = taxonomy.family_of_cwe(cwe)
|
||
|
|
if found and found.id == family:
|
||
|
|
return cwe
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def github_row(record: dict) -> dict:
|
||
|
|
"""An OSV GHSA record in the shape source.candidate expects (GitHub advisory API, so GitHub ecosystem names:
|
||
|
|
OSV's PyPI and Packagist become pip and composer)."""
|
||
|
|
specific = record.get("database_specific") or {}
|
||
|
|
return {
|
||
|
|
"ghsa_id": record["id"],
|
||
|
|
"published_at": record["published"],
|
||
|
|
"type": "reviewed" if specific.get("github_reviewed") else "unreviewed",
|
||
|
|
"withdrawn_at": record.get("withdrawn"),
|
||
|
|
"cwes": [{"cwe_id": c} for c in specific.get("cwe_ids", [])],
|
||
|
|
"vulnerabilities": [
|
||
|
|
{
|
||
|
|
"package": {
|
||
|
|
"ecosystem": OSV_ECOSYSTEMS.get(a["package"]["ecosystem"], a["package"]["ecosystem"]),
|
||
|
|
"name": a["package"]["name"],
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for a in record.get("affected", [])
|
||
|
|
if a.get("package")
|
||
|
|
],
|
||
|
|
"references": [r.get("url", "") for r in record.get("references", [])],
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
class ZipOSV:
|
||
|
|
"""source.OSV look-alike that answers from the downloaded dump (no network)."""
|
||
|
|
|
||
|
|
def __init__(self, records: dict[str, dict]):
|
||
|
|
self.records = records
|
||
|
|
self.calls = 0
|
||
|
|
|
||
|
|
def get(self, identifier: str) -> dict:
|
||
|
|
self.calls += 1
|
||
|
|
return self.records.get(identifier, {})
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_sources(args) -> int:
|
||
|
|
from benchmarks.unseen import source
|
||
|
|
from benchmarks.unseen.eligibility import RepositoryNotFound, normalize
|
||
|
|
from openultrasast.model.taxonomy import load_families
|
||
|
|
|
||
|
|
taxonomy = load_families()
|
||
|
|
accepted = pilot_accepted()
|
||
|
|
pilot = pilot_sources()
|
||
|
|
pilot_advisories = {row["advisory"] for row in pilot}
|
||
|
|
pilot_repos = {normalize(row["repository"]) for row in pilot}
|
||
|
|
public = pool_public()
|
||
|
|
pool_repos = {normalize(row["repository"]) for row in public}
|
||
|
|
used = set(json.loads((WORK / "used.json").read_text()))
|
||
|
|
excluded_repos = accepted | pilot_repos | pool_repos | used
|
||
|
|
log(
|
||
|
|
f"exclusion set: pilot accepted {len(accepted)}, pilot sourced {len(pilot_repos)}, pool public {len(pool_repos)}, used {len(used)} -> {len(excluded_repos)} names" # noqa: E501
|
||
|
|
)
|
||
|
|
|
||
|
|
osv_api = source.OSV()
|
||
|
|
rows: list[dict] = []
|
||
|
|
counts: Counter = Counter()
|
||
|
|
if args.extend:
|
||
|
|
# Second batch: only OSV, continuing the same seeded order past every repository already sourced.
|
||
|
|
earlier = json.loads((WORK / "sources.json").read_text())
|
||
|
|
excluded_repos |= {normalize(row["repository"]) for row in earlier}
|
||
|
|
log(f"extend: {len(earlier)} earlier sources excluded")
|
||
|
|
|
||
|
|
# (a) pool-fresh: permissive pool entries the pilot never sourced
|
||
|
|
for row in [] if args.extend else public:
|
||
|
|
repo = normalize(row["repository"])
|
||
|
|
if repo in accepted or row["advisory"] in pilot_advisories or repo in pilot_repos:
|
||
|
|
continue
|
||
|
|
record = osv_api.get(row["advisory"])
|
||
|
|
cwe = cwe_for(record, row["family"], taxonomy)
|
||
|
|
if not record or not cwe:
|
||
|
|
counts["pool-fresh:no_osv_record_or_cwe"] += 1
|
||
|
|
log(f"pool-fresh skip {row['advisory']}: no OSV record or no CWE in family {row['family']}")
|
||
|
|
continue
|
||
|
|
rows.append(
|
||
|
|
dict(
|
||
|
|
repository=repo,
|
||
|
|
repo=repo,
|
||
|
|
url=row["url"],
|
||
|
|
fix=row["fix"],
|
||
|
|
parent=row["parent"],
|
||
|
|
family=row["family"],
|
||
|
|
cwe=cwe,
|
||
|
|
advisory=row["advisory"],
|
||
|
|
advisory_text=osv_text(record),
|
||
|
|
ecosystem=row["ecosystem"],
|
||
|
|
license_spdx=row.get("license_spdx"),
|
||
|
|
origin="pool-fresh",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
counts["pool-fresh"] += 1
|
||
|
|
|
||
|
|
# (b) pool-retry: the pilot sourced them, no site passed, repository not held out
|
||
|
|
for row in [] if args.extend else pilot:
|
||
|
|
repo = normalize(row["repository"])
|
||
|
|
if repo in accepted:
|
||
|
|
continue
|
||
|
|
rows.append({**row, "repository": repo, "repo": repo, "origin": "pool-retry"})
|
||
|
|
counts["pool-retry"] += 1
|
||
|
|
|
||
|
|
# (c) OSV dump: GHSA advisories with exactly one GitHub fix commit, one family, repo not excluded
|
||
|
|
pre: dict[str, dict] = {}
|
||
|
|
records: dict[str, dict] = {}
|
||
|
|
for eco_name, eco in OSV_ECOSYSTEMS.items():
|
||
|
|
archive = zipfile.ZipFile(Path(args.osv_dir) / f"{eco_name}.zip")
|
||
|
|
names = [n for n in archive.namelist() if n.startswith("GHSA-") and n.endswith(".json")]
|
||
|
|
counts[f"osv:{eco}:ghsa_records"] += len(names)
|
||
|
|
for name in names:
|
||
|
|
record = json.loads(archive.read(name))
|
||
|
|
row = github_row(record)
|
||
|
|
try:
|
||
|
|
if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1):
|
||
|
|
raise source.Rejected("date")
|
||
|
|
if row["type"] != "reviewed" or row.get("withdrawn_at"):
|
||
|
|
raise source.Rejected("advisory_status")
|
||
|
|
if source.ecosystem(row) != eco:
|
||
|
|
raise source.Rejected("ecosystem")
|
||
|
|
families = {f.id for c in row["cwes"] if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in source.FAMILIES}
|
||
|
|
if len(families) != 1:
|
||
|
|
raise source.Rejected("family")
|
||
|
|
links = source.fix_links(row)
|
||
|
|
if len(links) != 1:
|
||
|
|
raise source.Rejected("fix_links")
|
||
|
|
((repo, _),) = links
|
||
|
|
if repo in excluded_repos:
|
||
|
|
raise source.Rejected("excluded_repository")
|
||
|
|
except source.Rejected as exc:
|
||
|
|
counts[f"osv:{eco}:{exc}"] += 1
|
||
|
|
continue
|
||
|
|
counts[f"osv:{eco}:prefilter_pass"] += 1
|
||
|
|
records[record["id"]] = record
|
||
|
|
# one advisory per repository: the most recently published one
|
||
|
|
current = pre.get(repo)
|
||
|
|
if current is None or row["published_at"] > current["published_at"]:
|
||
|
|
pre[repo] = {**row, "_eco": eco}
|
||
|
|
log(f"OSV prefilter: {len(pre)} repositories with a candidate advisory")
|
||
|
|
counts["osv:prefilter_repositories"] = len(pre)
|
||
|
|
|
||
|
|
token = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True, check=True).stdout.strip()
|
||
|
|
github = source.GitHub(token)
|
||
|
|
osv_stub = ZipOSV(records)
|
||
|
|
ordered = source.ordered_advisories(list(pre.values()), args.seed)
|
||
|
|
target = args.target
|
||
|
|
picked = 0
|
||
|
|
for row in ordered:
|
||
|
|
if picked >= target:
|
||
|
|
break
|
||
|
|
try:
|
||
|
|
cand = source.candidate(row, github, osv_stub)
|
||
|
|
except source.Rejected as exc:
|
||
|
|
counts[f"osv:check:{exc}"] += 1
|
||
|
|
continue
|
||
|
|
except RepositoryNotFound:
|
||
|
|
counts["osv:check:not_found"] += 1
|
||
|
|
continue
|
||
|
|
except ValueError as exc:
|
||
|
|
counts[f"osv:check:{type(exc).__name__}:{exc}"] += 1
|
||
|
|
continue
|
||
|
|
if cand.license_class != "permissive":
|
||
|
|
counts["osv:check:licence_not_permissive"] += 1
|
||
|
|
continue
|
||
|
|
if cand.repository in excluded_repos:
|
||
|
|
counts["osv:check:excluded_after_resolution"] += 1
|
||
|
|
continue
|
||
|
|
record = records[row["ghsa_id"]]
|
||
|
|
cwe = cwe_for(record, cand.family, taxonomy)
|
||
|
|
rows.append(
|
||
|
|
dict(
|
||
|
|
repository=cand.repository,
|
||
|
|
repo=cand.repository,
|
||
|
|
url=cand.url,
|
||
|
|
fix=cand.fix,
|
||
|
|
parent=cand.parent,
|
||
|
|
family=cand.family,
|
||
|
|
cwe=cwe,
|
||
|
|
advisory=cand.advisory,
|
||
|
|
advisory_text=osv_text(record),
|
||
|
|
ecosystem=cand.ecosystem,
|
||
|
|
license_spdx=cand.license_spdx,
|
||
|
|
origin="osv-extend" if args.extend else "osv",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
excluded_repos.add(cand.repository)
|
||
|
|
picked += 1
|
||
|
|
counts["osv"] += 1
|
||
|
|
log(f"osv pick {picked}/{target}: {cand.repository} {cand.advisory} {cand.ecosystem} {cand.family} {cand.license_spdx}")
|
||
|
|
counts["github_api_calls"] = github.calls
|
||
|
|
counts["osv_api_calls"] = osv_api.calls
|
||
|
|
# the candidate list must not contain a held-out repository, whatever the origin
|
||
|
|
assert not {row["repository"] for row in rows} & accepted
|
||
|
|
stem = "sources_extend" if args.extend else "sources"
|
||
|
|
(WORK / f"{stem}.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False) + "\n")
|
||
|
|
(WORK / f"{stem}_log.json").write_text(json.dumps(dict(sorted(counts.items())), indent=2) + "\n")
|
||
|
|
log(
|
||
|
|
json.dumps(
|
||
|
|
{k: v for k, v in counts.items() if not k.startswith("osv:") or "check" in k or "pass" in k or "repositories" in k},
|
||
|
|
sort_keys=True,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
log(f"sources.json: {len(rows)} advisories, {len({r['repository'] for r in rows})} repositories")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------- clone
|
||
|
|
|
||
|
|
|
||
|
|
def clone_one(repo: str, url: str, fix: str) -> tuple[Path | None, str, str | None]:
|
||
|
|
"""Clone single-branch with history, make sure fix and fix^1 are present, return (dir, status, parent)."""
|
||
|
|
target = HIST / repo.replace("/", "__")
|
||
|
|
if not (target / ".git").exists():
|
||
|
|
if target.exists():
|
||
|
|
shutil.rmtree(target)
|
||
|
|
done = subprocess.run(
|
||
|
|
["git", "-c", "credential.helper=", "clone", "--quiet", "--single-branch", "--no-tags", "--no-checkout", url, str(target)],
|
||
|
|
stdin=subprocess.DEVNULL,
|
||
|
|
capture_output=True,
|
||
|
|
text=True,
|
||
|
|
timeout=1800,
|
||
|
|
env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
|
||
|
|
)
|
||
|
|
if done.returncode:
|
||
|
|
shutil.rmtree(target, ignore_errors=True)
|
||
|
|
return None, "clone_failed", None
|
||
|
|
# No working tree: everything downstream reads objects (git show / log / diff). The first 64 clones were
|
||
|
|
# emptied the same way after the disk guard fired: `git sparse-checkout disable` restores a checkout.
|
||
|
|
git(target, "sparse-checkout", "set", "--no-cone", "/__worktree_emptied_for_disk__", check=False)
|
||
|
|
has_fix = git(target, "cat-file", "-e", f"{fix}^{{commit}}", check=False).returncode == 0
|
||
|
|
if not has_fix:
|
||
|
|
fetched = git(target, "fetch", "--quiet", "--depth", "2", "origin", fix, check=False)
|
||
|
|
if fetched.returncode:
|
||
|
|
fetched = git(target, "fetch", "--quiet", "origin", fix, check=False)
|
||
|
|
if fetched.returncode or git(target, "cat-file", "-e", f"{fix}^{{commit}}", check=False).returncode:
|
||
|
|
return target, "fix_unreachable", None
|
||
|
|
parents = git(target, "rev-list", "--parents", "-n", "1", fix).stdout.split()[1:]
|
||
|
|
if len(parents) != 1:
|
||
|
|
return target, f"parents_{len(parents)}", None
|
||
|
|
parent = parents[0]
|
||
|
|
if git(target, "cat-file", "-e", f"{parent}^{{commit}}", check=False).returncode:
|
||
|
|
fetched = git(target, "fetch", "--quiet", "--depth", "1", "origin", parent, check=False)
|
||
|
|
if fetched.returncode or git(target, "cat-file", "-e", f"{parent}^{{commit}}", check=False).returncode:
|
||
|
|
return target, "parent_unreachable", None
|
||
|
|
# prove the parent's tree is readable: the catalog reads blobs at fix^1
|
||
|
|
if git(target, "ls-tree", "-r", "--name-only", parent, check=False).returncode:
|
||
|
|
return target, "parent_tree_unreadable", None
|
||
|
|
return target, "ok", parent
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_clone(args) -> int:
|
||
|
|
sources_path = Path(args.sources) if args.sources else WORK / "sources.json"
|
||
|
|
rows = json.loads(sources_path.read_text())
|
||
|
|
HIST.mkdir(parents=True, exist_ok=True)
|
||
|
|
repos_path = WORK / "repos_hist.json"
|
||
|
|
repos = json.loads(repos_path.read_text()) if repos_path.exists() else {}
|
||
|
|
clone_log_path = WORK / "clone_log.json"
|
||
|
|
clone_log = json.loads(clone_log_path.read_text()) if clone_log_path.exists() else {}
|
||
|
|
by_repo: dict[str, list[dict]] = defaultdict(list)
|
||
|
|
for row in rows:
|
||
|
|
by_repo[row["repository"]].append(row)
|
||
|
|
stopped = False
|
||
|
|
for index, (repo, entries) in enumerate(sorted(by_repo.items()), 1):
|
||
|
|
if repo in clone_log and clone_log[repo].get("status") == "ok" and all(e.get("parent_from_git") for e in entries):
|
||
|
|
continue
|
||
|
|
free = free_bytes(HIST)
|
||
|
|
if free < MIN_FREE_BYTES:
|
||
|
|
log(f"STOP: free space {free / 1024**3:.1f} GB under 6 GB before cloning {repo}")
|
||
|
|
stopped = True
|
||
|
|
break
|
||
|
|
started = time.monotonic()
|
||
|
|
url = entries[0]["url"]
|
||
|
|
status, parent_seen = "ok", {}
|
||
|
|
target = None
|
||
|
|
for entry in entries:
|
||
|
|
target, status, parent = clone_one(repo, url, entry["fix"])
|
||
|
|
parent_seen[entry["fix"]] = parent
|
||
|
|
if status != "ok":
|
||
|
|
break
|
||
|
|
size = sum(p.stat().st_size for p in target.rglob("*") if p.is_file()) if target and target.exists() else 0
|
||
|
|
commits = (
|
||
|
|
git(target, "rev-list", "--count", "--first-parent", "HEAD", check=False).stdout.strip() if target and status == "ok" else ""
|
||
|
|
)
|
||
|
|
clone_log[repo] = {
|
||
|
|
"status": status,
|
||
|
|
"bytes": size,
|
||
|
|
"seconds": round(time.monotonic() - started, 1),
|
||
|
|
"first_parent_commits": int(commits) if commits else None,
|
||
|
|
"parents": parent_seen,
|
||
|
|
}
|
||
|
|
if status == "ok":
|
||
|
|
repos[repo] = str(target)
|
||
|
|
for entry in entries:
|
||
|
|
entry["parent_from_git"] = parent_seen[entry["fix"]]
|
||
|
|
else:
|
||
|
|
if target and target.exists():
|
||
|
|
shutil.rmtree(target, ignore_errors=True)
|
||
|
|
repos.pop(repo, None)
|
||
|
|
log(
|
||
|
|
f"[{index}/{len(by_repo)}] {repo}: {status} {size / 1024**2:.1f} MB {clone_log[repo]['seconds']}s free={free_bytes(HIST) / 1024**3:.1f}GB" # noqa: E501
|
||
|
|
)
|
||
|
|
repos_path.write_text(json.dumps(dict(sorted(repos.items())), indent=1) + "\n")
|
||
|
|
clone_log_path.write_text(json.dumps(dict(sorted(clone_log.items())), indent=1) + "\n")
|
||
|
|
# rewrite sources: parent = fix^1 from git; move uncloned entries aside
|
||
|
|
kept, dropped = [], []
|
||
|
|
for row in rows:
|
||
|
|
entry = clone_log.get(row["repository"], {})
|
||
|
|
parent = row.get("parent_from_git") or entry.get("parents", {}).get(row["fix"])
|
||
|
|
if entry.get("status") == "ok" and parent:
|
||
|
|
if row["parent"] != parent:
|
||
|
|
row["source_parent"] = row["parent"]
|
||
|
|
row["parent"] = parent
|
||
|
|
row.pop("parent_from_git", None)
|
||
|
|
kept.append(row)
|
||
|
|
else:
|
||
|
|
dropped.append({**row, "clone_status": entry.get("status", "not_cloned")})
|
||
|
|
sources_path.write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n")
|
||
|
|
(sources_path.with_name(sources_path.stem + "_dropped.json")).write_text(json.dumps(dropped, indent=1, ensure_ascii=False) + "\n")
|
||
|
|
total = sum(e["bytes"] for e in clone_log.values())
|
||
|
|
log(
|
||
|
|
f"cloned {sum(e['status'] == 'ok' for e in clone_log.values())}/{len(by_repo)} repositories, {total / 1024**3:.2f} GB, kept {len(kept)} sources, dropped {len(dropped)}, stopped={stopped}" # noqa: E501
|
||
|
|
)
|
||
|
|
return 1 if stopped else 0
|
||
|
|
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------- filter
|
||
|
|
|
||
|
|
|
||
|
|
def site_rejection(path: str) -> str | None:
|
||
|
|
if MINIFIED_PATH.search(path):
|
||
|
|
return "minified"
|
||
|
|
if VENDORED_PATH.search(path):
|
||
|
|
return "vendored"
|
||
|
|
if TEST_PATH.search(path):
|
||
|
|
return "test"
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_filter(args) -> int:
|
||
|
|
from benchmarks.valid.catalog import write_catalog
|
||
|
|
|
||
|
|
cases = tomllib.loads(Path(args.catalog).read_text())["case"]
|
||
|
|
kept, reasons = [], Counter()
|
||
|
|
for case in cases:
|
||
|
|
reason = site_rejection(case["path"])
|
||
|
|
if reason:
|
||
|
|
reasons[reason] += 1
|
||
|
|
continue
|
||
|
|
kept.append(case)
|
||
|
|
write_catalog(Path(args.out), kept)
|
||
|
|
log(f"catalog: {len(cases)} sites -> {len(kept)} kept, dropped {dict(reasons)}")
|
||
|
|
(WORK / "catalog_filter_log.json").write_text(
|
||
|
|
json.dumps({"input": len(cases), "kept": len(kept), "dropped": dict(reasons)}, indent=2) + "\n"
|
||
|
|
)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------- verify
|
||
|
|
|
||
|
|
|
||
|
|
class CountingClient:
|
||
|
|
def __init__(self, inner, budget: int):
|
||
|
|
self.inner, self.budget, self.calls = inner, budget, 0
|
||
|
|
|
||
|
|
def complete(self, **kwargs):
|
||
|
|
if self.calls >= self.budget:
|
||
|
|
raise BudgetExhausted()
|
||
|
|
self.calls += 1
|
||
|
|
return self.inner.complete(**kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
class BudgetExhausted(RuntimeError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_verify(args) -> int:
|
||
|
|
from benchmarks.unseen.eligibility import normalize
|
||
|
|
from benchmarks.valid.catalog import local_git
|
||
|
|
from benchmarks.valid.verify import DEFAULT_DETECTOR_MODEL, DeepSeekChatClient, verify_case, write_records
|
||
|
|
from openultrasast.config import load_config
|
||
|
|
from openultrasast.model.endpoint import resolve_chat_endpoint
|
||
|
|
|
||
|
|
if not os.environ.get("DEEPSEEK_API_KEY"):
|
||
|
|
log("DEEPSEEK_API_KEY is not exported")
|
||
|
|
return 2
|
||
|
|
resolved = resolve_chat_endpoint(load_config(dotenv=False))
|
||
|
|
if resolved is None or resolved[1].provider != "deepseek":
|
||
|
|
log("DeepSeek provider is required")
|
||
|
|
return 2
|
||
|
|
client = resolved[0]
|
||
|
|
if isinstance(client, DeepSeekChatClient):
|
||
|
|
client = client.bounded(1200)
|
||
|
|
counting = CountingClient(client, args.budget)
|
||
|
|
repos = {normalize(k): local_git(Path(v)) for k, v in json.loads(Path(args.repos).read_text()).items()}
|
||
|
|
cases = tomllib.loads(Path(args.catalog).read_text())["case"]
|
||
|
|
out = Path(args.out)
|
||
|
|
records = {row["id"]: row for row in json.loads(out.read_text())["cases"]} if out.exists() else {}
|
||
|
|
log_path = WORK / "verify_log.json"
|
||
|
|
vlog = json.loads(log_path.read_text()) if log_path.exists() else {"calls": 0, "errors": {}, "passes": 0}
|
||
|
|
# round-robin across advisories so an advisory with many sites cannot eat the budget alone
|
||
|
|
groups: dict[str, list[dict]] = defaultdict(list)
|
||
|
|
for case in sorted(cases, key=lambda c: c["id"]):
|
||
|
|
groups[case["advisory"]].append(case)
|
||
|
|
order = []
|
||
|
|
depth = max(len(v) for v in groups.values())
|
||
|
|
for level in range(depth):
|
||
|
|
for advisory in sorted(groups):
|
||
|
|
if level < len(groups[advisory]):
|
||
|
|
order.append(groups[advisory][level])
|
||
|
|
errors: Counter = Counter(vlog.get("errors", {}))
|
||
|
|
started = time.monotonic()
|
||
|
|
done = 0
|
||
|
|
for case in order:
|
||
|
|
if case["id"] in records and not (args.retry_errors and records[case["id"]].get("error")):
|
||
|
|
continue
|
||
|
|
if counting.calls >= args.budget:
|
||
|
|
log(f"budget of {args.budget} calls reached")
|
||
|
|
break
|
||
|
|
try:
|
||
|
|
record = verify_case(repos[normalize(case["repo"])], case, client=counting)
|
||
|
|
except BudgetExhausted:
|
||
|
|
break
|
||
|
|
except Exception as exc: # provider or parse failure: record the type only, never the payload
|
||
|
|
record = {
|
||
|
|
"id": case["id"],
|
||
|
|
"verdict": "rejected",
|
||
|
|
"error": type(exc).__name__,
|
||
|
|
"model": DEFAULT_DETECTOR_MODEL,
|
||
|
|
"human_spot_check": None,
|
||
|
|
}
|
||
|
|
errors[type(exc).__name__] += 1
|
||
|
|
records[case["id"]] = record
|
||
|
|
done += 1
|
||
|
|
passed = record.get("verdict") == "pending_human"
|
||
|
|
log(
|
||
|
|
f"{case['id']} -> {record.get('verdict')}{' PASS' if passed else ''}{' error=' + record['error'] if record.get('error') else ''} calls={counting.calls}" # noqa: E501
|
||
|
|
)
|
||
|
|
if done % 5 == 0:
|
||
|
|
write_records(out, list(records.values()))
|
||
|
|
write_records(out, list(records.values()))
|
||
|
|
vlog["calls"] = vlog.get("calls", 0) + counting.calls
|
||
|
|
vlog["errors"] = dict(errors)
|
||
|
|
vlog["passes"] = sum(r.get("verdict") == "pending_human" for r in records.values())
|
||
|
|
vlog["records"] = len(records)
|
||
|
|
vlog["catalog_sites"] = len(cases)
|
||
|
|
vlog["seconds"] = vlog.get("seconds", 0) + round(time.monotonic() - started)
|
||
|
|
vlog["model"] = DEFAULT_DETECTOR_MODEL
|
||
|
|
vlog["budget"] = args.budget
|
||
|
|
log_path.write_text(json.dumps(vlog, indent=2, sort_keys=True) + "\n")
|
||
|
|
log(json.dumps(vlog, sort_keys=True))
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------- candidates
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_candidates(args) -> int:
|
||
|
|
cases = {c["id"]: c for c in tomllib.loads(Path(args.catalog).read_text())["case"]}
|
||
|
|
records = json.loads(Path(args.verification).read_text())["cases"]
|
||
|
|
sources = {row["advisory"]: row for row in json.loads((HERE / "sources.json").read_text())}
|
||
|
|
passing = [r for r in records if r.get("verdict") == "pending_human"]
|
||
|
|
by_repo: dict[str, list[dict]] = defaultdict(list)
|
||
|
|
for record in passing:
|
||
|
|
by_repo[cases[record["id"]]["repo"]].append(record)
|
||
|
|
languages = Counter(cases[r["id"]]["language"] for r in passing)
|
||
|
|
changes = {(cases[r["id"]]["repo"], cases[r["id"]]["commit"]) for r in passing}
|
||
|
|
lines = [
|
||
|
|
"# Training-set candidates for the maintainer's spot-check",
|
||
|
|
"",
|
||
|
|
f"Generated {date.today().isoformat()} by `benchmarks/valid/train/sourcing.py candidates`. Every location below passed all three",
|
||
|
|
"model checks (vulnerable before the fix, fix addresses it, reachable from an in-repo entry point) and waits for a",
|
||
|
|
"human accept or reject. Nothing here is a positive until accepted. Accept or reject a location with",
|
||
|
|
"",
|
||
|
|
" .venv/bin/python -m benchmarks.valid.verify spot-check benchmarks/valid/train/verification.json <id> accept|reject --reviewer <you> --date YYYY-MM-DD", # noqa: E501
|
||
|
|
"",
|
||
|
|
f"Sites passing: {len(passing)} in {len(by_repo)} repositories, {len(changes)} positive changes (fix commits); languages: "
|
||
|
|
+ ", ".join(f"{k} {v}" for k, v in sorted(languages.items())),
|
||
|
|
"",
|
||
|
|
"The verifier's entry point is the model's claim, not a checked fact; it is the reason the site passed the reachability check.",
|
||
|
|
"",
|
||
|
|
]
|
||
|
|
for repo in sorted(by_repo):
|
||
|
|
rows = sorted(by_repo[repo], key=lambda r: r["id"])
|
||
|
|
first = cases[rows[0]["id"]]
|
||
|
|
src = sources.get(first["advisory"], {})
|
||
|
|
lines += [
|
||
|
|
f"## {repo}",
|
||
|
|
"",
|
||
|
|
f"- advisory: {first['advisory']} ({first['family']}, {first['cwe']}), origin `{src.get('origin', '?')}`, ecosystem {src.get('ecosystem', '?')}, licence {src.get('license_spdx', '?')}", # noqa: E501
|
||
|
|
f"- fix `{first['commit']}`, parent (fix^1) `{first['parent']}`",
|
||
|
|
f"- advisory title: {(first.get('advisory_text') or '').strip().splitlines()[0][:160] if first.get('advisory_text') else ''}",
|
||
|
|
"",
|
||
|
|
"| id | path::function (pre-fix lines) | language | entry point (verifier) | confidence |",
|
||
|
|
"|---|---|---|---|---|",
|
||
|
|
]
|
||
|
|
for record in rows:
|
||
|
|
case = cases[record["id"]]
|
||
|
|
span = case["function_lines"][0]
|
||
|
|
entry = str(record.get("entry_point", "")).replace("|", "\\|").replace("\n", " ")[:200]
|
||
|
|
lines.append(
|
||
|
|
f"| `{record['id']}` | `{case['path']}::{case['function']}` ({span[0]}-{span[1]}) | {case['language']} | {entry} | {record.get('confidence')} |" # noqa: E501
|
||
|
|
)
|
||
|
|
lines.append("")
|
||
|
|
Path(args.out).write_text("\n".join(lines) + "\n")
|
||
|
|
log(f"CANDIDATES.md: {len(passing)} sites, {len(by_repo)} repos, {len(changes)} changes, languages {dict(languages)}")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------- overlap
|
||
|
|
|
||
|
|
GITHUB = re.compile(r"github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", re.IGNORECASE)
|
||
|
|
REPO_LINE = re.compile(r'^\s*(?:repo|fix_repo)\s*=\s*"([^"]+)"', re.MULTILINE)
|
||
|
|
|
||
|
|
|
||
|
|
def named_elsewhere() -> dict[str, set[str]]:
|
||
|
|
"""Every repository the development tree already names, by where it is named. Mirrors the repository guards in
|
||
|
|
tests/ (test_advisory_fix_pairs, test_independent_population): github.com/<name> anywhere under benchmarks/, src/,
|
||
|
|
tests/, plane/ and .kiro/, plus `repo = "..."` lines, plus the strict used set. Vendored trees are skipped as the
|
||
|
|
pair guard skips them; this directory is skipped because it is what is being checked."""
|
||
|
|
from openultrasast.learn.labels import repo_name
|
||
|
|
|
||
|
|
found: dict[str, set[str]] = defaultdict(set)
|
||
|
|
for top in ("benchmarks", "src", "tests", "plane", ".kiro"):
|
||
|
|
for path in (ROOT / top).rglob("*"):
|
||
|
|
if "node_modules" in path.parts or "vendor" in path.parts or path.is_relative_to(HERE):
|
||
|
|
continue
|
||
|
|
if not path.is_file() or path.suffix in {".pyc", ".bin", ".gz"}:
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
text = path.read_text(errors="ignore")
|
||
|
|
except OSError:
|
||
|
|
continue
|
||
|
|
label = "benchmarks/" + path.relative_to(ROOT / "benchmarks").parts[0] if top == "benchmarks" else top
|
||
|
|
for match in GITHUB.findall(text):
|
||
|
|
found[repo_name(match)].add(label)
|
||
|
|
for match in REPO_LINE.findall(text):
|
||
|
|
with contextlib.suppress(Exception): # an unparseable value is not a repository name
|
||
|
|
found[repo_name(match)].add(label)
|
||
|
|
for name in json.loads((WORK / "used_strict.json").read_text()):
|
||
|
|
found[name].add("used_strict")
|
||
|
|
return found
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_overlap(args) -> int:
|
||
|
|
"""Drop every candidate whose repository the tree already names; write the surviving sources to the tree."""
|
||
|
|
from openultrasast.learn.labels import repo_name
|
||
|
|
|
||
|
|
elsewhere = named_elsewhere()
|
||
|
|
rows = [row for name in args.batches for row in json.loads((WORK / name).read_text())]
|
||
|
|
kept, dropped = [], []
|
||
|
|
for row in rows:
|
||
|
|
where = sorted(elsewhere.get(repo_name(row["repository"]), ()))
|
||
|
|
if where:
|
||
|
|
dropped.append({"repository": row["repository"], "advisory": row["advisory"], "origin": row["origin"], "named_in": where})
|
||
|
|
else:
|
||
|
|
kept.append(row)
|
||
|
|
assert not {repo_name(r["repository"]) for r in kept} & pilot_accepted()
|
||
|
|
(WORK / "overlap_dropped.json").write_text(json.dumps(dropped, indent=1) + "\n")
|
||
|
|
(WORK / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n")
|
||
|
|
(HERE / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n")
|
||
|
|
log(
|
||
|
|
f"overlap: {len(rows)} sources, {len(dropped)} dropped (named elsewhere), {len(kept)} kept; by origin dropped {dict(Counter(d['origin'] for d in dropped))}" # noqa: E501
|
||
|
|
)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_merge(args) -> int:
|
||
|
|
"""Tree catalog.toml and verification.json: the batch files restricted to the surviving sources."""
|
||
|
|
from benchmarks.valid.catalog import write_catalog
|
||
|
|
from benchmarks.valid.verify import write_records
|
||
|
|
|
||
|
|
advisories = {row["advisory"] for row in json.loads((HERE / "sources.json").read_text())}
|
||
|
|
cases = [c for name in args.catalogs for c in tomllib.loads((WORK / name).read_text()).get("case", []) if c["advisory"] in advisories]
|
||
|
|
# A site whose code or advisory text links a repository the tree already names (a pair-corpus or reserved
|
||
|
|
# repository in a code comment, say) would put that name back into benchmarks/: drop the site, record why.
|
||
|
|
from openultrasast.learn.labels import repo_name
|
||
|
|
|
||
|
|
elsewhere = json.loads((WORK / "named_elsewhere.json").read_text())
|
||
|
|
dropped = []
|
||
|
|
for case in list(cases):
|
||
|
|
names = {repo_name(m) for value in case.values() if isinstance(value, str) for m in GITHUB.findall(value)} - {case["repo"]}
|
||
|
|
hits = sorted(n for n in names if n in elsewhere and not n.startswith("user-attachments/"))
|
||
|
|
if hits:
|
||
|
|
cases.remove(case)
|
||
|
|
dropped.append({"id": case["id"], "repo": case["repo"], "names": hits})
|
||
|
|
(WORK / "merge_dropped_sites.json").write_text(json.dumps(dropped, indent=1) + "\n")
|
||
|
|
ids = {c["id"] for c in cases}
|
||
|
|
records = [r for name in args.verifications for r in json.loads((WORK / name).read_text())["cases"] if r["id"] in ids]
|
||
|
|
write_catalog(HERE / "catalog.toml", cases)
|
||
|
|
write_records(HERE / "verification.json", records)
|
||
|
|
log(f"merge: {len(cases)} sites, {len(records)} verification records, {len(dropped)} sites dropped for naming a used repository")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------- precheck
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_precheck(args) -> int:
|
||
|
|
from benchmarks.unseen import extract
|
||
|
|
from benchmarks.unseen.eligibility import normalize
|
||
|
|
from benchmarks.valid.catalog import local_git
|
||
|
|
|
||
|
|
cases = {c["id"]: c for c in tomllib.loads(Path(args.catalog).read_text())["case"]}
|
||
|
|
records = json.loads(Path(args.verification).read_text())["cases"]
|
||
|
|
repos = json.loads(Path(args.repos).read_text())
|
||
|
|
passing = defaultdict(list)
|
||
|
|
for record in records:
|
||
|
|
if record.get("verdict") == "pending_human":
|
||
|
|
passing[cases[record["id"]]["repo"]].append(cases[record["id"]])
|
||
|
|
result = {}
|
||
|
|
for repo in sorted(passing):
|
||
|
|
git_ = local_git(Path(repos[repo]))
|
||
|
|
all_cases = [c for c in cases.values() if normalize(c["repo"]) == repo]
|
||
|
|
fixes = {c["commit"] for c in all_cases}
|
||
|
|
functions = {(c["path"], c["function"]) for c in all_cases}
|
||
|
|
head = git_.run("rev-parse", "HEAD").strip()
|
||
|
|
started = time.monotonic()
|
||
|
|
try:
|
||
|
|
history = extract.ordinary_history(git_, head)
|
||
|
|
except Exception as exc: # assemble.freeze calls the same function and would stop on this repository
|
||
|
|
result[repo] = {"error": f"{type(exc).__name__}: {exc}", "seconds": round(time.monotonic() - started, 1)}
|
||
|
|
log(f"{repo}: ordinary_history failed: {type(exc).__name__}: {exc}")
|
||
|
|
(WORK / "precheck.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
||
|
|
continue
|
||
|
|
available = sum(extract.ordinary_rejection(row, fixes, functions) is None for row in history)
|
||
|
|
changes = len({(c["commit"], c["parent"]) for c in passing[repo]})
|
||
|
|
result[repo] = {
|
||
|
|
"first_parent_commits": len(history),
|
||
|
|
"ordinary_available": available,
|
||
|
|
"positive_changes": changes,
|
||
|
|
"needed_at_ratio_20": 20 * changes,
|
||
|
|
"enough": available >= 20 * changes,
|
||
|
|
"seconds": round(time.monotonic() - started, 1),
|
||
|
|
}
|
||
|
|
(WORK / "precheck.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
||
|
|
log(
|
||
|
|
f"{repo}: {available} ordinary of {len(history)} first-parent commits, needs {20 * changes}, enough={available >= 20 * changes}"
|
||
|
|
)
|
||
|
|
(WORK / "precheck.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def main(argv=None) -> int:
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
||
|
|
sub.add_parser("used")
|
||
|
|
p = sub.add_parser("sources")
|
||
|
|
p.add_argument("--osv-dir", required=True)
|
||
|
|
p.add_argument("--target", type=int, default=75, help="OSV advisories to accept after the GitHub checks")
|
||
|
|
p.add_argument("--seed", type=int, default=1)
|
||
|
|
p.add_argument("--extend", action="store_true", help="second OSV batch into sources_extend.json, excluding sources.json")
|
||
|
|
p = sub.add_parser("clone")
|
||
|
|
p.add_argument("--sources", help="sources file to clone (default sources.json)")
|
||
|
|
p = sub.add_parser("filter")
|
||
|
|
p.add_argument("catalog")
|
||
|
|
p.add_argument("--out", required=True)
|
||
|
|
p = sub.add_parser("verify")
|
||
|
|
p.add_argument("catalog")
|
||
|
|
p.add_argument("--repos", required=True)
|
||
|
|
p.add_argument("--out", required=True)
|
||
|
|
p.add_argument("--budget", type=int, default=600)
|
||
|
|
p.add_argument("--retry-errors", action="store_true")
|
||
|
|
p = sub.add_parser("candidates")
|
||
|
|
p.add_argument("verification")
|
||
|
|
p.add_argument("catalog")
|
||
|
|
p.add_argument("--out", required=True)
|
||
|
|
p = sub.add_parser("overlap")
|
||
|
|
p.add_argument("batches", nargs="+", help="source batch files in the work directory")
|
||
|
|
p = sub.add_parser("merge")
|
||
|
|
p.add_argument("--catalogs", nargs="+", required=True)
|
||
|
|
p.add_argument("--verifications", nargs="+", required=True)
|
||
|
|
p = sub.add_parser("precheck")
|
||
|
|
p.add_argument("verification")
|
||
|
|
p.add_argument("catalog")
|
||
|
|
p.add_argument("--repos", required=True)
|
||
|
|
args = parser.parse_args(argv)
|
||
|
|
return {
|
||
|
|
"used": cmd_used,
|
||
|
|
"sources": cmd_sources,
|
||
|
|
"clone": cmd_clone,
|
||
|
|
"filter": cmd_filter,
|
||
|
|
"verify": cmd_verify,
|
||
|
|
"candidates": cmd_candidates,
|
||
|
|
"precheck": cmd_precheck,
|
||
|
|
"overlap": cmd_overlap,
|
||
|
|
"merge": cmd_merge,
|
||
|
|
}[args.command](args)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
sys.exit(main())
|