bench(train): class-targeted sourcing, --family, from the public OSV dumps

sources --family <family> keeps only advisories whose CWE the taxonomy maps to that family, writes
sources_<family>.json beside the training set, and excludes the pilot, every repository the
benchmarks already name, the strict used set and every earlier batch; --prefilter-only runs the
dump pass without GitHub. Implemented by Codex (gpt-6-astra), gated here: names normalize rejects
exclude nothing instead of aborting, and the strict used set is in the exclusions.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/valid/train/SOURCING.md

@@ -192,3 +192,35 @@ python $B candidates benchmarks/valid/train/verification.json benchmarks/valid/t
 
 The verifier ran before the overlap check, so 137 of the 553 calls went to locations that were dropped later
 (134 by the overlap check, 3 by the merge). A fresh run should call `overlap` before `clone`.
+
+## Grow one family
+
+To source another access-control batch from the existing ecosystem dumps:
+
+```sh
+.venv/bin/python -m benchmarks.valid.train.sourcing sources \
+  --family access_control --osv-dir /path/to/dumps --target 120
+```
+
+The directory must contain `PyPI.zip`, `npm.zip`, `Maven.zip` and `Packagist.zip`. The family comes from
+`taxonomy.family_of_cwe`; the command prints its CWE ids at startup. Family runs use only OSV records,
+keep the existing single-family rule, and exclude the strict used set, the pilot's accepted repositories,
+every repository named elsewhere in the development tree, and repositories in `sources.json` and
+`sources_batch*.json` in both the work and training directories. These exclusions happen before GitHub checks.
+
+The target defaults to 120 accepted advisories for a family run, or 75 without `--family`. Acceptance still
+requires the existing GitHub repository, commit-parent and licence checks. Family output goes to
+`benchmarks/valid/train/sources_access_control.json` and `sources_access_control_log.json`; the earlier
+source files are unchanged. Repeating a family run replaces that family's output. `--family` and `--extend`
+are mutually exclusive. Cloning, catalog building and verification remain separate maintainer steps.
+
+For a network-free dump inspection, add `--prefilter-only`. It reads the same dumps and exclusions, prints
+per-ecosystem record, rejection and prefilter counts, and writes an empty source list with `accepted: 0`
+and `github_checks_run: 0` in the log. Prefilter passes are not accepted advisories. Missing exclusion files
+or unreadable dumps fail the run rather than silently weakening the exclusions.
+
+`overlap --family access_control <batch>` also writes only that family's source file; a batch can be an
+absolute path to the family output. `merge --family access_control --catalogs ... --verifications ...`
+uses that family's sources and writes `catalog_access_control.toml` and `verification_access_control.json`.
+Both commands keep their drop logs in the work directory with the family in the filename. Use the same
+`--family` throughout to preserve the original training batch.

benchmarks/valid/train/sourcing.py

@@ -179,20 +179,56 @@ class ZipOSV:
         return self.records.get(identifier, {})
 
 
+def family_exclusions() -> dict[str, set[str]]:
+    """Collect strict, pilot, tree and earlier batch exclusions before GitHub checks."""
+    from benchmarks.unseen.eligibility import normalize
+
+    def normalized(names) -> set[str]:
+        out = set()
+        for name in names:
+            try:
+                out.add(normalize(name))
+            except ValueError:
+                continue  # a link fragment that is not a repository name excludes nothing
+        return out
+
+    groups = {
+        "pilot": normalized(pilot_accepted()),
+        "named_elsewhere": normalized(named_elsewhere()),
+        "used_strict": normalized(json.loads((WORK / "used_strict.json").read_text())) if (WORK / "used_strict.json").is_file() else set(),
+        "already_sourced": set(),
+    }
+    for directory in (WORK, HERE):
+        paths = {directory / "sources.json", *directory.glob("sources_batch*.json")}
+        for path in sorted(paths):
+            if not path.is_file() or path.stem.endswith("_log"):
+                continue
+            groups["already_sourced"].update(normalize(row["repository"]) for row in json.loads(path.read_text()))
+    return groups
+
+
 def cmd_sources(args) -> int:
     from benchmarks.unseen import source
     from benchmarks.unseen.eligibility import RepositoryNotFound, normalize
     from openultrasast.model.taxonomy import load_families
 
     taxonomy = load_families()
-    accepted = pilot_accepted()
-    pilot = pilot_sources()
+    family = args.family
+    if family:
+        log(f"family {family}: {', '.join(sorted(taxonomy.by_id(family).cwes))}")
+    accepted = {normalize(name) for name in pilot_accepted()}
+    pilot = [] if family else pilot_sources()
     pilot_advisories = {row["advisory"] for row in pilot}
     pilot_repos = {normalize(row["repository"]) for row in pilot}
-    public = pool_public()
+    public = [] if family else pool_public()
     pool_repos = {normalize(row["repository"]) for row in public}
-    used = set(json.loads((WORK / "used.json").read_text()))
+    used = set() if family else set(json.loads((WORK / "used.json").read_text()))
     excluded_repos = accepted | pilot_repos | pool_repos | used
+    exclusion_groups = family_exclusions() if family else {"repository": excluded_repos}
+    excluded_repos |= set().union(*exclusion_groups.values())
+    if family:
+        log(f"exclusion set: { {reason: len(names) for reason, names in exclusion_groups.items()} } -> {len(excluded_repos)} names")
+    else:
         log(
             f"exclusion set: pilot accepted {len(accepted)}, pilot sourced {len(pilot_repos)}, pool public {len(pool_repos)}, used {len(used)} -> {len(excluded_repos)} names"  # noqa: E501
         )
@@ -200,7 +236,7 @@ def cmd_sources(args) -> int:
     osv_api = source.OSV()
     rows: list[dict] = []
     counts: Counter = Counter()
-    if args.extend:
+    if args.extend and not family:
         # Second batch: only OSV, continuing the same seeded order past every repository already sourced.
         earlier = json.loads((WORK / "sources.json").read_text())
         excluded_repos |= {normalize(row["repository"]) for row in earlier}
@@ -247,11 +283,11 @@ def cmd_sources(args) -> int:
     pre: dict[str, dict] = {}
     records: dict[str, dict] = {}
     for eco_name, eco in OSV_ECOSYSTEMS.items():
-        archive = zipfile.ZipFile(Path(args.osv_dir) / f"{eco_name}.zip")
-        names = [n for n in archive.namelist() if n.startswith("GHSA-") and n.endswith(".json")]
-        counts[f"osv:{eco}:ghsa_records"] += len(names)
-        for name in names:
-            record = json.loads(archive.read(name))
+        archive_path = Path(args.osv_dir) / f"{eco_name}.zip"
+        counts[f"osv:{eco}:ghsa_records"] = 0
+        counts[f"osv:{eco}:prefilter_pass"] = 0
+        for record in dump_records(archive_path):
+            counts[f"osv:{eco}:ghsa_records"] += 1
             row = github_row(record)
             try:
                 if date.fromisoformat(row["published_at"][:10]) < date(2019, 1, 1):
@@ -263,12 +299,15 @@ def cmd_sources(args) -> int:
                 families = {f.id for c in row["cwes"] if (f := taxonomy.family_of_cwe(c["cwe_id"])) and f.id in source.FAMILIES}
                 if len(families) != 1:
                     raise source.Rejected("family")
+                if family and families != {family}:
+                    raise source.Rejected("family_mismatch")
                 links = source.fix_links(row)
                 if len(links) != 1:
                     raise source.Rejected("fix_links")
                 ((repo, _),) = links
                 if repo in excluded_repos:
-                    raise source.Rejected("excluded_repository")
+                    reason = next(reason for reason, names in exclusion_groups.items() if repo in names)
+                    raise source.Rejected(f"excluded_{reason}")
             except source.Rejected as exc:
                 counts[f"osv:{eco}:{exc}"] += 1
                 continue
@@ -278,16 +317,24 @@ def cmd_sources(args) -> int:
             current = pre.get(repo)
             if current is None or row["published_at"] > current["published_at"]:
                 pre[repo] = {**row, "_eco": eco}
+        log(json.dumps({k: v for k, v in sorted(counts.items()) if k.startswith(f"osv:{eco}:")}))
     log(f"OSV prefilter: {len(pre)} repositories with a candidate advisory")
     counts["osv:prefilter_repositories"] = len(pre)
 
+    github = None
+    if not args.prefilter_only:
         token = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True, check=True).stdout.strip()
         github = source.GitHub(token)
+    else:
+        log("PREFILTER ONLY: GitHub checks did not run; no advisories are accepted.")
     osv_stub = ZipOSV(records)
     ordered = source.ordered_advisories(list(pre.values()), args.seed)
-    target = args.target
+    target = args.target if args.target is not None else (120 if family else 75)
+    counts["target"] = target
+    counts["github_checks_run"] = int(github is not None)
+    counts["accepted"] = 0
     picked = 0
-    for row in ordered:
+    for row in ordered if github is not None else []:
         if picked >= target:
             break
         try:
@@ -328,24 +375,35 @@ def cmd_sources(args) -> int:
         excluded_repos.add(cand.repository)
         picked += 1
         counts["osv"] += 1
+        counts["accepted"] += 1
         log(f"osv pick {picked}/{target}: {cand.repository} {cand.advisory} {cand.ecosystem} {cand.family} {cand.license_spdx}")
-    counts["github_api_calls"] = github.calls
+    counts["github_api_calls"] = github.calls if github is not None else 0
     counts["osv_api_calls"] = osv_api.calls
     # the candidate list must not contain a held-out repository, whatever the origin
     assert not {row["repository"] for row in rows} & accepted
-    stem = "sources_extend" if args.extend else "sources"
-    (WORK / f"{stem}.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False) + "\n")
-    (WORK / f"{stem}_log.json").write_text(json.dumps(dict(sorted(counts.items())), indent=2) + "\n")
-    log(
-        json.dumps(
-            {k: v for k, v in counts.items() if not k.startswith("osv:") or "check" in k or "pass" in k or "repositories" in k},
-            sort_keys=True,
-        )
-    )
-    log(f"sources.json: {len(rows)} advisories, {len({r['repository'] for r in rows})} repositories")
+    stem = f"sources_{family}" if family else ("sources_extend" if args.extend else "sources")
+    output = HERE if family else WORK
+    (output / f"{stem}.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False) + "\n")
+    (output / f"{stem}_log.json").write_text(json.dumps(dict(sorted(counts.items())), indent=2) + "\n")
+    log(json.dumps(dict(sorted(counts.items()))))
+    log(f"{stem}.json: {len(rows)} advisories, {len({r['repository'] for r in rows})} repositories")
     return 0
 
 
+def dump_records(path: Path):
+    """Read GHSA records from a dump and report the bytes read from the first record."""
+    with zipfile.ZipFile(path) as archive:
+        first = True
+        for name in archive.namelist():
+            if not name.startswith("GHSA-") or not name.endswith(".json"):
+                continue
+            data = archive.read(name)
+            if first:
+                log(f"read {path}: {name}, {len(data)} bytes")
+                first = False
+            yield json.loads(data)
+
+
 # ----------------------------------------------------------------------------- clone
 
 
@@ -688,18 +746,26 @@ def cmd_overlap(args) -> int:
     from openultrasast.learn.labels import repo_name
 
     elsewhere = named_elsewhere()
+    if args.family:
+        for reason, names in family_exclusions().items():
+            for name in names:
+                elsewhere.setdefault(name, set()).add(reason)
     rows = [row for name in args.batches for row in json.loads((WORK / name).read_text())]
     kept, dropped = [], []
     for row in rows:
+        if args.family and row["family"] != args.family:
+            continue
         where = sorted(elsewhere.get(repo_name(row["repository"]), ()))
         if where:
             dropped.append({"repository": row["repository"], "advisory": row["advisory"], "origin": row["origin"], "named_in": where})
         else:
             kept.append(row)
     assert not {repo_name(r["repository"]) for r in kept} & pilot_accepted()
-    (WORK / "overlap_dropped.json").write_text(json.dumps(dropped, indent=1) + "\n")
+    suffix = f"_{args.family}" if args.family else ""
+    (WORK / f"overlap{suffix}_dropped.json").write_text(json.dumps(dropped, indent=1) + "\n")
+    if not args.family:
         (WORK / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n")
-    (HERE / "sources.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n")
+    (HERE / f"sources{suffix}.json").write_text(json.dumps(kept, indent=1, ensure_ascii=False) + "\n")
     log(
         f"overlap: {len(rows)} sources, {len(dropped)} dropped (named elsewhere), {len(kept)} kept; by origin dropped {dict(Counter(d['origin'] for d in dropped))}"  # noqa: E501
     )
@@ -711,7 +777,8 @@ def cmd_merge(args) -> int:
     from benchmarks.valid.catalog import write_catalog
     from benchmarks.valid.verify import write_records
 
-    advisories = {row["advisory"] for row in json.loads((HERE / "sources.json").read_text())}
+    suffix = f"_{args.family}" if args.family else ""
+    advisories = {row["advisory"] for row in json.loads((HERE / f"sources{suffix}.json").read_text())}
     cases = [c for name in args.catalogs for c in tomllib.loads((WORK / name).read_text()).get("case", []) if c["advisory"] in advisories]
     # A site whose code or advisory text links a repository the tree already names (a pair-corpus or reserved
     # repository in a code comment, say) would put that name back into benchmarks/: drop the site, record why.
@@ -725,11 +792,11 @@ def cmd_merge(args) -> int:
         if hits:
             cases.remove(case)
             dropped.append({"id": case["id"], "repo": case["repo"], "names": hits})
-    (WORK / "merge_dropped_sites.json").write_text(json.dumps(dropped, indent=1) + "\n")
+    (WORK / f"merge{suffix}_dropped_sites.json").write_text(json.dumps(dropped, indent=1) + "\n")
     ids = {c["id"] for c in cases}
     records = [r for name in args.verifications for r in json.loads((WORK / name).read_text())["cases"] if r["id"] in ids]
-    write_catalog(HERE / "catalog.toml", cases)
-    write_records(HERE / "verification.json", records)
+    write_catalog(HERE / f"catalog{suffix}.toml", cases)
+    write_records(HERE / f"verification{suffix}.json", records)
     log(f"merge: {len(cases)} sites, {len(records)} verification records, {len(dropped)} sites dropped for naming a used repository")
     return 0
 
@@ -783,14 +850,21 @@ def cmd_precheck(args) -> int:
 
 
 def main(argv=None) -> int:
+    from benchmarks.unseen.source import FAMILIES
+
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     sub = parser.add_subparsers(dest="command", required=True)
     sub.add_parser("used")
     p = sub.add_parser("sources")
     p.add_argument("--osv-dir", required=True)
-    p.add_argument("--target", type=int, default=75, help="OSV advisories to accept after the GitHub checks")
+    p.add_argument("--target", type=int, help="OSV advisories to accept after GitHub checks (default: 120 with --family, otherwise 75)")
     p.add_argument("--seed", type=int, default=1)
-    p.add_argument("--extend", action="store_true", help="second OSV batch into sources_extend.json, excluding sources.json")
+    mode = p.add_mutually_exclusive_group()
+    mode.add_argument("--extend", action="store_true", help="second OSV batch into sources_extend.json, excluding sources.json")
+    mode.add_argument("--family", choices=sorted(FAMILIES), help="source one family into sources_<family>.json in the training directory")
+    p.add_argument(
+        "--prefilter-only", action="store_true", help="with --family, read dumps and exclusions without GitHub checks or accepts"
+    )
     p = sub.add_parser("clone")
     p.add_argument("--sources", help="sources file to clone (default sources.json)")
     p = sub.add_parser("filter")
@@ -807,8 +881,10 @@ def main(argv=None) -> int:
     p.add_argument("catalog")
     p.add_argument("--out", required=True)
     p = sub.add_parser("overlap")
+    p.add_argument("--family", choices=sorted(FAMILIES))
     p.add_argument("batches", nargs="+", help="source batch files in the work directory")
     p = sub.add_parser("merge")
+    p.add_argument("--family", choices=sorted(FAMILIES))
     p.add_argument("--catalogs", nargs="+", required=True)
     p.add_argument("--verifications", nargs="+", required=True)
     p = sub.add_parser("precheck")
@@ -816,6 +892,8 @@ def main(argv=None) -> int:
     p.add_argument("catalog")
     p.add_argument("--repos", required=True)
     args = parser.parse_args(argv)
+    if args.command == "sources" and args.prefilter_only and not args.family:
+        parser.error("--prefilter-only requires --family")
     return {
         "used": cmd_used,
         "sources": cmd_sources,

tests/test_train_sourcing.py

@@ -0,0 +1,192 @@
+"""Offline tests for family-specific training sources."""
+
+import importlib
+import json
+import socket
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+sourcing = importlib.import_module("benchmarks.valid.train.sourcing")
+source = importlib.import_module("benchmarks.unseen.source")
+
+
+@pytest.fixture(autouse=True)
+def offline(monkeypatch):
+    def forbidden(*args, **kwargs):
+        pytest.fail("unexpected network or subprocess call")
+
+    monkeypatch.setattr(socket, "create_connection", forbidden)
+    monkeypatch.setattr(socket.socket, "connect", forbidden)
+    monkeypatch.setattr(subprocess, "run", forbidden)
+
+
+@pytest.fixture
+def environment(tmp_path, monkeypatch):
+    root = tmp_path / "root"
+    here = root / "benchmarks/valid/train"
+    work = tmp_path / "work"
+    pilot = root / "benchmarks/pilot"
+    dumps = tmp_path / "dumps"
+    for directory in (here, work, pilot, dumps):
+        directory.mkdir(parents=True)
+    for name, value in (("ROOT", root), ("HERE", here), ("WORK", work), ("PILOT", pilot)):
+        monkeypatch.setattr(sourcing, name, value)
+    (work / "used_strict.json").write_text(json.dumps(["fixture/used"]))
+    (pilot / "accepted_repos.json").write_text("[]")
+    (here / "sources.json").write_text('[{"repository": "fixture/earlier"}]\n')
+    (work / "sources.json").write_text("[]\n")
+
+    def record(name, cwes):
+        return {
+            "id": f"GHSA-{name}",
+            "published": "2025-01-01T00:00:00Z",
+            "database_specific": {"github_reviewed": True, "cwe_ids": cwes},
+            "affected": [{"package": {"ecosystem": "PyPI", "name": name}}],
+            "references": [{"url": f"https://github.com/fixture/{name}/commit/" + "a" * 40}],
+        }
+
+    records = [record("allowed", ["CWE-862"]), record("injection", ["CWE-89"]), record("no-cwe", []), record("used", ["CWE-862"])]
+
+    def write_dumps(rows):
+        for ecosystem in sourcing.OSV_ECOSYSTEMS:
+            with zipfile.ZipFile(dumps / f"{ecosystem}.zip", "w") as archive:
+                for row in rows if ecosystem == "PyPI" else []:
+                    archive.writestr(row["id"] + ".json", json.dumps(row))
+
+    write_dumps(records)
+    return here, work, pilot, dumps, records, record, write_dumps
+
+
+@pytest.fixture
+def github(monkeypatch):
+    class GitHub:
+        calls = 0
+        repositories = []
+
+        def __init__(self, token):
+            assert token == "fixture-token"
+
+        def get_repo(self, name):
+            self.calls += 1
+            self.repositories.append(name)
+            return {"private": False, "size": 100, "clone_url": f"https://github.com/{name}.git", "full_name": name}
+
+        def get_commit(self, name, sha):
+            self.calls += 1
+            return {"sha": sha, "parents": [{"sha": "b" * 40}]}
+
+        def get_license(self, name, ref):
+            self.calls += 1
+            return {"spdx_id": "MIT", "path": "LICENSE"}
+
+    def token(argv, **kwargs):
+        assert argv == ["gh", "auth", "token"]
+        return subprocess.CompletedProcess(argv, 0, stdout="fixture-token\n")
+
+    monkeypatch.setattr(subprocess, "run", token)
+    monkeypatch.setattr(source, "GitHub", GitHub)
+    return GitHub
+
+
+def run_sources(dumps, *extra):
+    return sourcing.main(["sources", "--family", "access_control", "--osv-dir", str(dumps), *extra])
+
+
+def test_family_filters_zip_and_preserves_sources(environment, github, capsys):
+    here, work, _, dumps, *_ = environment
+    original = (here / "sources.json").read_bytes()
+    work_original = (work / "sources.json").read_bytes()
+    assert run_sources(dumps) == 0
+    rows = json.loads((here / "sources_access_control.json").read_text())
+    assert [(row["repository"], row["family"], row["cwe"]) for row in rows] == [("fixture/allowed", "access_control", "CWE-862")]
+    assert github.repositories == ["fixture/allowed"]
+    assert (here / "sources.json").read_bytes() == original
+    assert (work / "sources.json").read_bytes() == work_original
+    counts = json.loads((here / "sources_access_control_log.json").read_text())
+    assert counts["osv:pip:ghsa_records"] == 4
+    assert counts["osv:pip:prefilter_pass"] == counts["accepted"] == 1
+    assert counts["osv:pip:family"] == counts["osv:pip:family_mismatch"] == counts["osv:pip:excluded_named_elsewhere"] == 1
+    assert counts["target"] == 120
+    assert counts["github_api_calls"] == 3
+    output = capsys.readouterr().out
+    assert "bytes" in output and '"osv:pip:ghsa_records": 4' in output
+
+
+def test_unknown_family_fails_before_io(tmp_path, capsys):
+    with pytest.raises(SystemExit) as error:
+        sourcing.main(["sources", "--family", "not-a-family", "--osv-dir", str(tmp_path)])
+    assert error.value.code == 2
+    assert "invalid choice" in capsys.readouterr().err
+    assert not list(tmp_path.iterdir())
+
+
+def test_prefilter_only_never_accepts_or_calls_github(environment, capsys):
+    here, _, _, dumps, *_ = environment
+    assert run_sources(dumps, "--prefilter-only", "--target", "120") == 0
+    assert json.loads((here / "sources_access_control.json").read_text()) == []
+    counts = json.loads((here / "sources_access_control_log.json").read_text())
+    assert counts["osv:pip:prefilter_pass"] == 1
+    assert counts["accepted"] == counts["github_api_calls"] == counts["osv_api_calls"] == counts["github_checks_run"] == 0
+    assert "GitHub checks did not run" in capsys.readouterr().out
+
+
+def test_all_exclusions_apply_before_github(environment, github):
+    here, work, pilot, dumps, records, record, write_dumps = environment
+    (pilot / "accepted_repos.json").write_text('["FIXTURE/Held-out"]')
+    (pilot / "named.md").write_text("https://github.com/fixture/named\n")
+    (work / "sources.json").write_text('[{"repository": "fixture/work-source"}]')
+    (work / "sources_batch1.json").write_text('[{"repository": "FIXTURE/Batch"}]')
+    (work / "sources_batch1_log.json").write_text('{"accepted": 1}')
+    records += [record(name, ["CWE-862"]) for name in ("held-out", "named", "earlier", "work-source", "batch")]
+    records.append(record("mixed", ["CWE-862", "CWE-89"]))
+    write_dumps(records)
+    assert run_sources(dumps) == 0
+    assert github.repositories == ["fixture/allowed"]
+    counts = json.loads((here / "sources_access_control_log.json").read_text())
+    assert counts["osv:pip:excluded_pilot"] == 1
+    assert counts["osv:pip:excluded_named_elsewhere"] == 2
+    assert counts["osv:pip:excluded_already_sourced"] == 3
+    assert counts["osv:pip:family"] == 2
+
+
+def test_target_counts_accepts_after_github_rejection(environment, github, monkeypatch):
+    here, _, _, dumps, records, record, write_dumps = environment
+    records.extend(record(f"extra-{i}", ["CWE-862"]) for i in range(3))
+    write_dumps(records)
+    attempts = []
+
+    def license_record(self, name, ref):
+        attempts.append(name)
+        return {"spdx_id": "NONE" if len(attempts) == 1 else "MIT", "path": "LICENSE"}
+
+    monkeypatch.setattr(github, "get_license", license_record)
+    assert run_sources(dumps, "--target", "2") == 0
+    assert len(attempts) == 3
+    counts = json.loads((here / "sources_access_control_log.json").read_text())
+    assert counts["accepted"] == counts["target"] == 2
+    assert counts["osv:check:licence_not_permissive"] == 1
+
+
+def test_family_overlap_and_merge_keep_batches_separate(environment):
+    here, work, _, _, *_ = environment
+    original = (here / "sources.json").read_bytes()
+    batch = [{"repository": "fixture/new", "advisory": "GHSA-new", "origin": "osv", "family": "access_control"}]
+    (work / "new.json").write_text(json.dumps(batch))
+    assert sourcing.main(["overlap", "--family", "access_control", "new.json"]) == 0
+    assert json.loads((here / "sources_access_control.json").read_text()) == batch
+    (work / "named_elsewhere.json").write_text("{}")
+    (work / "catalog.toml").write_text("case = []\n")
+    (work / "verification.json").write_text('{"cases": []}')
+    (here / "catalog.toml").write_text("original catalog")
+    (here / "verification.json").write_text("original verification")
+    assert sourcing.main(["merge", "--family", "access_control", "--catalogs", "catalog.toml", "--verifications", "verification.json"]) == 0
+    assert (here / "catalog_access_control.toml").is_file()
+    assert (here / "verification_access_control.json").is_file()
+    assert (here / "sources.json").read_bytes() == original
+    assert (here / "catalog.toml").read_text() == "original catalog"
+    assert (here / "verification.json").read_text() == "original verification"