From 3d08d692ff3066fea066e0f14005dcff1f3db736 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:46:45 +0200 Subject: [PATCH] bench(train): class-targeted sourcing, --family, from the public OSV dumps sources --family keeps only advisories whose CWE the taxonomy maps to that family, writes sources_.json beside the training set, and excludes the pilot, every repository the benchmarks already name, the strict used set and every earlier batch; --prefilter-only runs the dump pass without GitHub. Implemented by Codex (gpt-6-astra), gated here: names normalize rejects exclude nothing instead of aborting, and the strict used set is in the exclusions. Co-Authored-By: Claude Fable 5.1 --- benchmarks/valid/train/SOURCING.md | 32 +++++ benchmarks/valid/train/sourcing.py | 154 +++++++++++++++++------ tests/test_train_sourcing.py | 192 +++++++++++++++++++++++++++++ 3 files changed, 340 insertions(+), 38 deletions(-) create mode 100644 tests/test_train_sourcing.py diff --git a/benchmarks/valid/train/SOURCING.md b/benchmarks/valid/train/SOURCING.md index 823006e..fb00c99 100644 --- a/benchmarks/valid/train/SOURCING.md +++ b/benchmarks/valid/train/SOURCING.md @@ -192,3 +192,35 @@ python $B candidates benchmarks/valid/train/verification.json benchmarks/valid/t The verifier ran before the overlap check, so 137 of the 553 calls went to locations that were dropped later (134 by the overlap check, 3 by the merge). A fresh run should call `overlap` before `clone`. + +## Grow one family + +To source another access-control batch from the existing ecosystem dumps: + +```sh +.venv/bin/python -m benchmarks.valid.train.sourcing sources \ + --family access_control --osv-dir /path/to/dumps --target 120 +``` + +The directory must contain `PyPI.zip`, `npm.zip`, `Maven.zip` and `Packagist.zip`. The family comes from +`taxonomy.family_of_cwe`; the command prints its CWE ids at startup. Family runs use only OSV records, +keep the existing single-family rule, and exclude the strict used set, the pilot's accepted repositories, +every repository named elsewhere in the development tree, and repositories in `sources.json` and +`sources_batch*.json` in both the work and training directories. These exclusions happen before GitHub checks. + +The target defaults to 120 accepted advisories for a family run, or 75 without `--family`. Acceptance still +requires the existing GitHub repository, commit-parent and licence checks. Family output goes to +`benchmarks/valid/train/sources_access_control.json` and `sources_access_control_log.json`; the earlier +source files are unchanged. Repeating a family run replaces that family's output. `--family` and `--extend` +are mutually exclusive. Cloning, catalog building and verification remain separate maintainer steps. + +For a network-free dump inspection, add `--prefilter-only`. It reads the same dumps and exclusions, prints +per-ecosystem record, rejection and prefilter counts, and writes an empty source list with `accepted: 0` +and `github_checks_run: 0` in the log. Prefilter passes are not accepted advisories. Missing exclusion files +or unreadable dumps fail the run rather than silently weakening the exclusions. + +`overlap --family access_control ` also writes only that family's source file; a batch can be an +absolute path to the family output. `merge --family access_control --catalogs ... --verifications ...` +uses that family's sources and writes `catalog_access_control.toml` and `verification_access_control.json`. +Both commands keep their drop logs in the work directory with the family in the filename. Use the same +`--family` throughout to preserve the original training batch. diff --git a/benchmarks/valid/train/sourcing.py b/benchmarks/valid/train/sourcing.py index 733fb77..91bd949 100644 --- a/benchmarks/valid/train/sourcing.py +++ b/benchmarks/valid/train/sourcing.py @@ -179,28 +179,64 @@ class ZipOSV: return self.records.get(identifier, {}) +def family_exclusions() -> dict[str, set[str]]: + """Collect strict, pilot, tree and earlier batch exclusions before GitHub checks.""" + from benchmarks.unseen.eligibility import normalize + + def normalized(names) -> set[str]: + out = set() + for name in names: + try: + out.add(normalize(name)) + except ValueError: + continue # a link fragment that is not a repository name excludes nothing + return out + + groups = { + "pilot": normalized(pilot_accepted()), + "named_elsewhere": normalized(named_elsewhere()), + "used_strict": normalized(json.loads((WORK / "used_strict.json").read_text())) if (WORK / "used_strict.json").is_file() else set(), + "already_sourced": set(), + } + for directory in (WORK, HERE): + paths = {directory / "sources.json", *directory.glob("sources_batch*.json")} + for path in sorted(paths): + if not path.is_file() or path.stem.endswith("_log"): + continue + groups["already_sourced"].update(normalize(row["repository"]) for row in json.loads(path.read_text())) + return groups + + def cmd_sources(args) -> int: from benchmarks.unseen import source from benchmarks.unseen.eligibility import RepositoryNotFound, normalize from openultrasast.model.taxonomy import load_families taxonomy = load_families() - accepted = pilot_accepted() - pilot = pilot_sources() + family = args.family + if family: + log(f"family {family}: {', '.join(sorted(taxonomy.by_id(family).cwes))}") + accepted = {normalize(name) for name in pilot_accepted()} + pilot = [] if family else pilot_sources() pilot_advisories = {row["advisory"] for row in pilot} pilot_repos = {normalize(row["repository"]) for row in pilot} - public = pool_public() + public = [] if family else pool_public() pool_repos = {normalize(row["repository"]) for row in public} - used = set(json.loads((WORK / "used.json").read_text())) + used = set() if family else set(json.loads((WORK / "used.json").read_text())) excluded_repos = accepted | pilot_repos | pool_repos | used - log( - f"exclusion set: pilot accepted {len(accepted)}, pilot sourced {len(pilot_repos)}, pool public {len(pool_repos)}, used {len(used)} -> {len(excluded_repos)} names" # noqa: E501 - ) + exclusion_groups = family_exclusions() if family else {"repository": excluded_repos} + excluded_repos |= set().union(*exclusion_groups.values()) + if family: + log(f"exclusion set: { {reason: len(names) for reason, names in exclusion_groups.items()} } -> {len(excluded_repos)} names") + else: + log( + f"exclusion set: pilot accepted {len(accepted)}, pilot sourced {len(pilot_repos)}, pool public {len(pool_repos)}, used {len(used)} -> {len(excluded_repos)} names" # noqa: E501 + ) osv_api = source.OSV() rows: list[dict] = [] counts: Counter = Counter() - if args.extend: + if args.extend and not family: # Second batch: only OSV, continuing the same seeded order past every repository already sourced. earlier = json.loads((WORK / "sources.json").read_text()) excluded_repos |= {normalize(row["repository"]) for row in earlier} @@ -247,11 +283,11 @@ def cmd_sources(args) -> int: pre: dict[str, dict] = {} records: dict[str, dict] = {} for eco_name, eco in OSV_ECOSYSTEMS.items(): - archive = zipfile.ZipFile(Path(args.osv_dir) / f"{eco_name}.zip") - names = [n for n in archive.namelist() if n.startswith("GHSA-") and n.endswith(".json")] - counts[f"osv:{eco}:ghsa_records"] += len(names) - for name in names: - record = json.loads(archive.read(name)) + archive_path = Path(args.osv_dir) / f"{eco_name}.zip" + counts[f"osv:{eco}:ghsa_records"] = 0 + counts[f"osv:{eco}:prefilter_pass"] = 0 + for record in dump_records(archive_path): + counts[f"osv:{eco}:ghsa_records"] += 1 row = github_row(record) try: if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1): @@ -263,12 +299,15 @@ def cmd_sources(args) -> int: families = {f.id for c in row["cwes"] if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in source.FAMILIES} if len(families) != 1: raise source.Rejected("family") + if family and families != {family}: + raise source.Rejected("family_mismatch") links = source.fix_links(row) if len(links) != 1: raise source.Rejected("fix_links") ((repo, _),) = links if repo in excluded_repos: - raise source.Rejected("excluded_repository") + reason = next(reason for reason, names in exclusion_groups.items() if repo in names) + raise source.Rejected(f"excluded_{reason}") except source.Rejected as exc: counts[f"osv:{eco}:{exc}"] += 1 continue @@ -278,16 +317,24 @@ def cmd_sources(args) -> int: current = pre.get(repo) if current is None or row["published_at"] > current["published_at"]: pre[repo] = {**row, "_eco": eco} + log(json.dumps({k: v for k, v in sorted(counts.items()) if k.startswith(f"osv:{eco}:")})) log(f"OSV prefilter: {len(pre)} repositories with a candidate advisory") counts["osv:prefilter_repositories"] = len(pre) - token = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True, check=True).stdout.strip() - github = source.GitHub(token) + github = None + if not args.prefilter_only: + token = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True, check=True).stdout.strip() + github = source.GitHub(token) + else: + log("PREFILTER ONLY: GitHub checks did not run; no advisories are accepted.") osv_stub = ZipOSV(records) ordered = source.ordered_advisories(list(pre.values()), args.seed) - target = args.target + target = args.target if args.target is not None else (120 if family else 75) + counts["target"] = target + counts["github_checks_run"] = int(github is not None) + counts["accepted"] = 0 picked = 0 - for row in ordered: + for row in ordered if github is not None else []: if picked >= target: break try: @@ -328,24 +375,35 @@ def cmd_sources(args) -> int: excluded_repos.add(cand.repository) picked += 1 counts["osv"] += 1 + counts["accepted"] += 1 log(f"osv pick {picked}/{target}: {cand.repository} {cand.advisory} {cand.ecosystem} {cand.family} {cand.license_spdx}") - counts["github_api_calls"] = github.calls + counts["github_api_calls"] = github.calls if github is not None else 0 counts["osv_api_calls"] = osv_api.calls # the candidate list must not contain a held-out repository, whatever the origin assert not {row["repository"] for row in rows} & accepted - stem = "sources_extend" if args.extend else "sources" - (WORK / f"{stem}.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False) + "\n") - (WORK / f"{stem}_log.json").write_text(json.dumps(dict(sorted(counts.items())), indent=2) + "\n") - log( - json.dumps( - {k: v for k, v in counts.items() if not k.startswith("osv:") or "check" in k or "pass" in k or "repositories" in k}, - sort_keys=True, - ) - ) - log(f"sources.json: {len(rows)} advisories, {len({r['repository'] for r in rows})} repositories") + stem = f"sources_{family}" if family else ("sources_extend" if args.extend else "sources") + output = HERE if family else WORK + (output / f"{stem}.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False) + "\n") + (output / f"{stem}_log.json").write_text(json.dumps(dict(sorted(counts.items())), indent=2) + "\n") + log(json.dumps(dict(sorted(counts.items())))) + log(f"{stem}.json: {len(rows)} advisories, {len({r['repository'] for r in rows})} repositories") return 0 +def dump_records(path: Path): + """Read GHSA records from a dump and report the bytes read from the first record.""" + with zipfile.ZipFile(path) as archive: + first = True + for name in archive.namelist(): + if not name.startswith("GHSA-") or not name.endswith(".json"): + continue + data = archive.read(name) + if first: + log(f"read {path}: {name}, {len(data)} bytes") + first = False + yield json.loads(data) + + # ----------------------------------------------------------------------------- clone @@ -688,18 +746,26 @@ def cmd_overlap(args) -> int: from openultrasast.learn.labels import repo_name elsewhere = named_elsewhere() + if args.family: + for reason, names in family_exclusions().items(): + for name in names: + elsewhere.setdefault(name, set()).add(reason) rows = [row for name in args.batches for row in json.loads((WORK / name).read_text())] kept, dropped = [], [] for row in rows: + if args.family and row["family"] != args.family: + continue where = sorted(elsewhere.get(repo_name(row["repository"]), ())) if where: dropped.append({"repository": row["repository"], "advisory": row["advisory"], "origin": row["origin"], "named_in": where}) else: kept.append(row) assert not {repo_name(r["repository"]) for r in kept} & pilot_accepted() - (WORK / "overlap_dropped.json").write_text(json.dumps(dropped, indent=1) + "\n") - (WORK / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n") - (HERE / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n") + suffix = f"_{args.family}" if args.family else "" + (WORK / f"overlap{suffix}_dropped.json").write_text(json.dumps(dropped, indent=1) + "\n") + if not args.family: + (WORK / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n") + (HERE / f"sources{suffix}.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n") log( f"overlap: {len(rows)} sources, {len(dropped)} dropped (named elsewhere), {len(kept)} kept; by origin dropped {dict(Counter(d['origin'] for d in dropped))}" # noqa: E501 ) @@ -711,7 +777,8 @@ def cmd_merge(args) -> int: from benchmarks.valid.catalog import write_catalog from benchmarks.valid.verify import write_records - advisories = {row["advisory"] for row in json.loads((HERE / "sources.json").read_text())} + suffix = f"_{args.family}" if args.family else "" + advisories = {row["advisory"] for row in json.loads((HERE / f"sources{suffix}.json").read_text())} cases = [c for name in args.catalogs for c in tomllib.loads((WORK / name).read_text()).get("case", []) if c["advisory"] in advisories] # A site whose code or advisory text links a repository the tree already names (a pair-corpus or reserved # repository in a code comment, say) would put that name back into benchmarks/: drop the site, record why. @@ -725,11 +792,11 @@ def cmd_merge(args) -> int: if hits: cases.remove(case) dropped.append({"id": case["id"], "repo": case["repo"], "names": hits}) - (WORK / "merge_dropped_sites.json").write_text(json.dumps(dropped, indent=1) + "\n") + (WORK / f"merge{suffix}_dropped_sites.json").write_text(json.dumps(dropped, indent=1) + "\n") ids = {c["id"] for c in cases} records = [r for name in args.verifications for r in json.loads((WORK / name).read_text())["cases"] if r["id"] in ids] - write_catalog(HERE / "catalog.toml", cases) - write_records(HERE / "verification.json", records) + write_catalog(HERE / f"catalog{suffix}.toml", cases) + write_records(HERE / f"verification{suffix}.json", records) log(f"merge: {len(cases)} sites, {len(records)} verification records, {len(dropped)} sites dropped for naming a used repository") return 0 @@ -783,14 +850,21 @@ def cmd_precheck(args) -> int: def main(argv=None) -> int: + from benchmarks.unseen.source import FAMILIES + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("used") p = sub.add_parser("sources") p.add_argument("--osv-dir", required=True) - p.add_argument("--target", type=int, default=75, help="OSV advisories to accept after the GitHub checks") + p.add_argument("--target", type=int, help="OSV advisories to accept after GitHub checks (default: 120 with --family, otherwise 75)") p.add_argument("--seed", type=int, default=1) - p.add_argument("--extend", action="store_true", help="second OSV batch into sources_extend.json, excluding sources.json") + mode = p.add_mutually_exclusive_group() + mode.add_argument("--extend", action="store_true", help="second OSV batch into sources_extend.json, excluding sources.json") + mode.add_argument("--family", choices=sorted(FAMILIES), help="source one family into sources_.json in the training directory") + p.add_argument( + "--prefilter-only", action="store_true", help="with --family, read dumps and exclusions without GitHub checks or accepts" + ) p = sub.add_parser("clone") p.add_argument("--sources", help="sources file to clone (default sources.json)") p = sub.add_parser("filter") @@ -807,8 +881,10 @@ def main(argv=None) -> int: p.add_argument("catalog") p.add_argument("--out", required=True) p = sub.add_parser("overlap") + p.add_argument("--family", choices=sorted(FAMILIES)) p.add_argument("batches", nargs="+", help="source batch files in the work directory") p = sub.add_parser("merge") + p.add_argument("--family", choices=sorted(FAMILIES)) p.add_argument("--catalogs", nargs="+", required=True) p.add_argument("--verifications", nargs="+", required=True) p = sub.add_parser("precheck") @@ -816,6 +892,8 @@ def main(argv=None) -> int: p.add_argument("catalog") p.add_argument("--repos", required=True) args = parser.parse_args(argv) + if args.command == "sources" and args.prefilter_only and not args.family: + parser.error("--prefilter-only requires --family") return { "used": cmd_used, "sources": cmd_sources, diff --git a/tests/test_train_sourcing.py b/tests/test_train_sourcing.py new file mode 100644 index 0000000..8c13f62 --- /dev/null +++ b/tests/test_train_sourcing.py @@ -0,0 +1,192 @@ +"""Offline tests for family-specific training sources.""" + +import importlib +import json +import socket +import subprocess +import sys +import zipfile +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sourcing = importlib.import_module("benchmarks.valid.train.sourcing") +source = importlib.import_module("benchmarks.unseen.source") + + +@pytest.fixture(autouse=True) +def offline(monkeypatch): + def forbidden(*args, **kwargs): + pytest.fail("unexpected network or subprocess call") + + monkeypatch.setattr(socket, "create_connection", forbidden) + monkeypatch.setattr(socket.socket, "connect", forbidden) + monkeypatch.setattr(subprocess, "run", forbidden) + + +@pytest.fixture +def environment(tmp_path, monkeypatch): + root = tmp_path / "root" + here = root / "benchmarks/valid/train" + work = tmp_path / "work" + pilot = root / "benchmarks/pilot" + dumps = tmp_path / "dumps" + for directory in (here, work, pilot, dumps): + directory.mkdir(parents=True) + for name, value in (("ROOT", root), ("HERE", here), ("WORK", work), ("PILOT", pilot)): + monkeypatch.setattr(sourcing, name, value) + (work / "used_strict.json").write_text(json.dumps(["fixture/used"])) + (pilot / "accepted_repos.json").write_text("[]") + (here / "sources.json").write_text('[{"repository": "fixture/earlier"}]\n') + (work / "sources.json").write_text("[]\n") + + def record(name, cwes): + return { + "id": f"GHSA-{name}", + "published": "2025-01-01T00:00:00Z", + "database_specific": {"github_reviewed": True, "cwe_ids": cwes}, + "affected": [{"package": {"ecosystem": "PyPI", "name": name}}], + "references": [{"url": f"https://github.com/fixture/{name}/commit/" + "a" * 40}], + } + + records = [record("allowed", ["CWE-862"]), record("injection", ["CWE-89"]), record("no-cwe", []), record("used", ["CWE-862"])] + + def write_dumps(rows): + for ecosystem in sourcing.OSV_ECOSYSTEMS: + with zipfile.ZipFile(dumps / f"{ecosystem}.zip", "w") as archive: + for row in rows if ecosystem == "PyPI" else []: + archive.writestr(row["id"] + ".json", json.dumps(row)) + + write_dumps(records) + return here, work, pilot, dumps, records, record, write_dumps + + +@pytest.fixture +def github(monkeypatch): + class GitHub: + calls = 0 + repositories = [] + + def __init__(self, token): + assert token == "fixture-token" + + def get_repo(self, name): + self.calls += 1 + self.repositories.append(name) + return {"private": False, "size": 100, "clone_url": f"https://github.com/{name}.git", "full_name": name} + + def get_commit(self, name, sha): + self.calls += 1 + return {"sha": sha, "parents": [{"sha": "b" * 40}]} + + def get_license(self, name, ref): + self.calls += 1 + return {"spdx_id": "MIT", "path": "LICENSE"} + + def token(argv, **kwargs): + assert argv == ["gh", "auth", "token"] + return subprocess.CompletedProcess(argv, 0, stdout="fixture-token\n") + + monkeypatch.setattr(subprocess, "run", token) + monkeypatch.setattr(source, "GitHub", GitHub) + return GitHub + + +def run_sources(dumps, *extra): + return sourcing.main(["sources", "--family", "access_control", "--osv-dir", str(dumps), *extra]) + + +def test_family_filters_zip_and_preserves_sources(environment, github, capsys): + here, work, _, dumps, *_ = environment + original = (here / "sources.json").read_bytes() + work_original = (work / "sources.json").read_bytes() + assert run_sources(dumps) == 0 + rows = json.loads((here / "sources_access_control.json").read_text()) + assert [(row["repository"], row["family"], row["cwe"]) for row in rows] == [("fixture/allowed", "access_control", "CWE-862")] + assert github.repositories == ["fixture/allowed"] + assert (here / "sources.json").read_bytes() == original + assert (work / "sources.json").read_bytes() == work_original + counts = json.loads((here / "sources_access_control_log.json").read_text()) + assert counts["osv:pip:ghsa_records"] == 4 + assert counts["osv:pip:prefilter_pass"] == counts["accepted"] == 1 + assert counts["osv:pip:family"] == counts["osv:pip:family_mismatch"] == counts["osv:pip:excluded_named_elsewhere"] == 1 + assert counts["target"] == 120 + assert counts["github_api_calls"] == 3 + output = capsys.readouterr().out + assert "bytes" in output and '"osv:pip:ghsa_records": 4' in output + + +def test_unknown_family_fails_before_io(tmp_path, capsys): + with pytest.raises(SystemExit) as error: + sourcing.main(["sources", "--family", "not-a-family", "--osv-dir", str(tmp_path)]) + assert error.value.code == 2 + assert "invalid choice" in capsys.readouterr().err + assert not list(tmp_path.iterdir()) + + +def test_prefilter_only_never_accepts_or_calls_github(environment, capsys): + here, _, _, dumps, *_ = environment + assert run_sources(dumps, "--prefilter-only", "--target", "120") == 0 + assert json.loads((here / "sources_access_control.json").read_text()) == [] + counts = json.loads((here / "sources_access_control_log.json").read_text()) + assert counts["osv:pip:prefilter_pass"] == 1 + assert counts["accepted"] == counts["github_api_calls"] == counts["osv_api_calls"] == counts["github_checks_run"] == 0 + assert "GitHub checks did not run" in capsys.readouterr().out + + +def test_all_exclusions_apply_before_github(environment, github): + here, work, pilot, dumps, records, record, write_dumps = environment + (pilot / "accepted_repos.json").write_text('["FIXTURE/Held-out"]') + (pilot / "named.md").write_text("https://github.com/fixture/named\n") + (work / "sources.json").write_text('[{"repository": "fixture/work-source"}]') + (work / "sources_batch1.json").write_text('[{"repository": "FIXTURE/Batch"}]') + (work / "sources_batch1_log.json").write_text('{"accepted": 1}') + records += [record(name, ["CWE-862"]) for name in ("held-out", "named", "earlier", "work-source", "batch")] + records.append(record("mixed", ["CWE-862", "CWE-89"])) + write_dumps(records) + assert run_sources(dumps) == 0 + assert github.repositories == ["fixture/allowed"] + counts = json.loads((here / "sources_access_control_log.json").read_text()) + assert counts["osv:pip:excluded_pilot"] == 1 + assert counts["osv:pip:excluded_named_elsewhere"] == 2 + assert counts["osv:pip:excluded_already_sourced"] == 3 + assert counts["osv:pip:family"] == 2 + + +def test_target_counts_accepts_after_github_rejection(environment, github, monkeypatch): + here, _, _, dumps, records, record, write_dumps = environment + records.extend(record(f"extra-{i}", ["CWE-862"]) for i in range(3)) + write_dumps(records) + attempts = [] + + def license_record(self, name, ref): + attempts.append(name) + return {"spdx_id": "NONE" if len(attempts) == 1 else "MIT", "path": "LICENSE"} + + monkeypatch.setattr(github, "get_license", license_record) + assert run_sources(dumps, "--target", "2") == 0 + assert len(attempts) == 3 + counts = json.loads((here / "sources_access_control_log.json").read_text()) + assert counts["accepted"] == counts["target"] == 2 + assert counts["osv:check:licence_not_permissive"] == 1 + + +def test_family_overlap_and_merge_keep_batches_separate(environment): + here, work, _, _, *_ = environment + original = (here / "sources.json").read_bytes() + batch = [{"repository": "fixture/new", "advisory": "GHSA-new", "origin": "osv", "family": "access_control"}] + (work / "new.json").write_text(json.dumps(batch)) + assert sourcing.main(["overlap", "--family", "access_control", "new.json"]) == 0 + assert json.loads((here / "sources_access_control.json").read_text()) == batch + (work / "named_elsewhere.json").write_text("{}") + (work / "catalog.toml").write_text("case = []\n") + (work / "verification.json").write_text('{"cases": []}') + (here / "catalog.toml").write_text("original catalog") + (here / "verification.json").write_text("original verification") + assert sourcing.main(["merge", "--family", "access_control", "--catalogs", "catalog.toml", "--verifications", "verification.json"]) == 0 + assert (here / "catalog_access_control.toml").is_file() + assert (here / "verification_access_control.json").is_file() + assert (here / "sources.json").read_bytes() == original + assert (here / "catalog.toml").read_text() == "original catalog" + assert (here / "verification.json").read_text() == "original verification"