unseen-eval group 1: used set (1,029 repositories, every source read), extended guard, usage ledger; record 01

Implemented with Codex; task 1.5 run by the coordinator against the real
sources (S3 memory read-only).

Co-Authored-By: Claude Opus 5.5 <[email protected]>

benchmarks/unseen/eligibility.py

@@ -0,0 +1,379 @@
+"""Read-only used-set collection. Only counts, source labels and digests leave this module's CLI.
+
+Run from the repository root: python -m benchmarks.unseen.eligibility --help.
+Population v3 is exclusively the existing guard's responsibility.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import tomllib
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import yaml
+
+from openultrasast.plane.memory import MemoryStore
+
+URL = re.compile(r"github\.com/([\w.-]+/[\w.-]+)", re.I)
+NAME = re.compile(r"[\w.-]+/[\w.-]+")
+SOURCES = ("pairs", "pointers", "populations", "harvest", "units", "measurements", "sweep")
+
+
+def normalize(value: str) -> str:
+    text = value.strip().lower().rstrip("/")
+    if NAME.fullmatch(text):
+        name = text.removesuffix(".git")
+    else:
+        # Structured fields must be identities, not prose containing a URL.
+        if any(char.isspace() for char in text):
+            raise ValueError("invalid_repository")
+        url = urlsplit(text if "://" in text else "https://" + text)
+        if url.scheme not in {"http", "https", "ssh", "git"} or not url.hostname or "." not in url.hostname:
+            raise ValueError("invalid_repository")
+        parts = url.path.strip("/").split("/")
+        if len(parts) < 2:
+            raise ValueError("invalid_repository")
+        name = "/".join(parts[:2]).removesuffix(".git")
+        if not NAME.fullmatch(name):
+            raise ValueError("invalid_repository")
+        if url.hostname != "github.com":
+            return url.hostname + "/" + name
+    if not NAME.fullmatch(name) or any(part in {".", ".."} for part in name.split("/")):
+        raise ValueError("invalid_repository")
+    return name
+
+
+def excluded(path: Path) -> bool:
+    return any("v3" in part.lower() or part in {".git", "__pycache__", ".venv"} for part in path.parts)
+
+
+def repositories(value: object, counts: dict[str, int] | None = None) -> set[str]:
+    """Structured repo fields include bare owner/name pointers; URLs also occur in prose."""
+    found = set()
+    if isinstance(value, dict):
+        for key, item in value.items():
+            if key in {"repo", "fix_repo", "repository", "full_name", "origin"} or (key == "group" and "unit" in value):
+                try:
+                    if not isinstance(item, str):
+                        raise ValueError("invalid_repository")
+                    found.add(normalize(item))
+                except ValueError:
+                    if counts is not None:
+                        counts["skipped_values"] = counts.get("skipped_values", 0) + 1
+            else:
+                found.update(repositories(item, counts))
+    elif isinstance(value, list):
+        for item in value:
+            found.update(repositories(item, counts))
+    elif isinstance(value, str):
+        found.update(normalize(m[1]) for m in URL.finditer(value))
+    return found
+
+
+def pointer_repositories(value: object, counts: dict[str, int] | None = None) -> set[str]:
+    if isinstance(value, dict):
+        if value.get("vendored") is False:
+            return repositories(value, counts)
+        return set().union(*(pointer_repositories(v, counts) for v in value.values()))
+    if isinstance(value, list):
+        return set().union(*(pointer_repositories(v, counts) for v in value))
+    return set()
+
+
+def parse(path: Path, raw: bytes) -> object:
+    structured = path.suffix in {".toml", ".json", ".jsonl", ".yaml", ".yml"}
+    text = raw.decode("utf-8", errors="strict" if structured else "ignore")
+    if path.suffix == ".toml":
+        return tomllib.loads(text)
+    if path.suffix == ".json":
+        return json.loads(text)
+    if path.suffix == ".jsonl":
+        return [json.loads(line) for line in text.splitlines() if line.strip()]
+    if path.suffix in {".yaml", ".yml"}:
+        documents = list(yaml.safe_load_all(text))
+        return documents[0] if len(documents) == 1 else documents
+    return text
+
+
+@dataclass
+class UsedSet:
+    sources: dict[str, set[str]] = field(default_factory=lambda: {label: set() for label in SOURCES})
+    instrument: dict[str, dict[str, int]] = field(default_factory=dict)
+
+    def add(self, label: str, names: set[str], size: int, files: int = 1, skipped_values: int = 0) -> None:
+        self.sources.setdefault(label, set()).update(names)
+        counts = self.instrument.setdefault(label, {"files": 0, "bytes": 0, "rows": 0, "skipped_values": 0})
+        counts["files"] += files
+        counts["bytes"] += size
+        counts["rows"] += len(names)
+        counts["skipped_values"] += skipped_values
+
+    def report(self, known_empty: set[str] | None = None) -> dict:
+        known_empty = known_empty or set()
+        if known_empty - self.sources.keys():
+            raise ValueError("unknown_source_label")
+        records = {}
+        for label, names in self.sources.items():
+            counts = self.instrument.get(label, {"files": 0, "bytes": 0, "rows": 0, "skipped_values": 0})
+            if not counts["rows"] and label not in known_empty:
+                raise ValueError("empty_source:" + label)
+            records[label] = {**counts, "count": len(names), "known_empty": label in known_empty}
+        names = set().union(*self.sources.values())
+        return {"sources": records, "count": len(names), "sha256": hashlib.sha256("\n".join(sorted(names)).encode()).hexdigest()}
+
+
+def build_used(root: Path, *, stores: Mapping[str, MemoryStore] | None = None, pointer_manifests: Iterable[Path] = ()) -> UsedSet:
+    used = UsedSet()
+
+    def read(label: str, paths: Iterable[Path], structured: bool = True) -> list[object]:
+        values = []
+        for path in sorted(set(paths)):
+            if excluded(path):
+                continue
+            raw = path.read_bytes()  # failure is an instrument failure, never a quiet empty source
+            value = parse(path, raw) if structured else raw.decode("utf-8", errors="ignore")
+            counts: dict[str, int] = {}
+            names = repositories(value, counts)
+            used.add(label, names, len(raw), skipped_values=counts.get("skipped_values", 0))
+            if label == "pairs":
+                pointer_counts: dict[str, int] = {}
+                pointers = pointer_repositories(value, pointer_counts)
+                if pointers or pointer_counts:
+                    used.add("pointers", pointers, len(raw), skipped_values=pointer_counts.get("skipped_values", 0))
+            values.append(value)
+        return values
+
+    pairs = root / "benchmarks/pairs"
+    read("pairs", (p for p in pairs.rglob("*") if p.is_file() and p.suffix in {".toml", ".json", ".jsonl", ".csv", ".yaml", ".yml"}))
+    read("pointers", pointer_manifests)
+    read("populations", (p for p in (root / "benchmarks/independent").glob("population-v[12].toml")))
+    read("harvest", (p for directory in pairs.glob("advisory-fixes*") for p in directory.rglob("*.toml")))
+    units = set((root / "plane/experiments").glob("*.units.jsonl"))
+    for path in (root / "plane/experiments").glob("*.y*ml"):
+        if excluded(path):
+            continue
+        raw = path.read_bytes()
+        manifest = parse(path, raw)
+        counts = {}
+        names = repositories(manifest, counts)
+        used.add("units", names, len(raw), skipped_values=counts.get("skipped_values", 0))
+        if isinstance(manifest, dict) and manifest.get("units_file"):
+            declared = root / Path(manifest["units_file"]).expanduser()
+            if declared.exists():  # an unregistered plan may not have frozen units yet
+                units.add(declared)
+    for label, store in (stores or {}).items():
+        if label not in {"local", "s3"}:
+            raise ValueError("unknown_store_label")
+        source = "memory_" + label
+        used.add(source, set(), 0, 0)
+        # Read actual objects, not a serialized row size masquerading as bytes read.
+        for key in store._keys("repos/"):
+            if not key.endswith(".jsonl") or excluded(Path(key)):
+                continue
+            result = store._get(key)
+            if result is None:
+                raise ValueError("missing_memory_object")
+            raw = result[0]
+            rows = parse(Path("rows.jsonl"), raw)
+            names = set()
+            counts = {}
+            for row in rows:
+                if row.get("kind") == "example" and "repo" in row:
+                    names.update(repositories({"repo": row["repo"]}, counts))
+                if row.get("kind") == "experiment" and row.get("units_file"):
+                    registered = root / Path(row["units_file"]).expanduser()
+                    if row.get("units_digest") or registered.exists():
+                        units.add(registered)
+            used.add(source, names, len(raw), skipped_values=counts.get("skipped_values", 0))
+    read("units", units)
+    read("measurements", (p for top in ("measurements", "experiments") for p in (root / "benchmarks" / top).rglob("*") if p.is_file()))
+    read(
+        "sweep",
+        (
+            p
+            for top in ("benchmarks", "src", "tests", "plane", ".kiro")
+            for p in (root / top).rglob("*")
+            if p.is_file() and not p.is_relative_to(root / "benchmarks/unseen") and not excluded(p)
+        ),
+        structured=False,
+    )
+    return used
+
+
+@dataclass(frozen=True)
+class Resolution:
+    full_name: str
+    names: frozenset[str]
+    network: str
+
+
+def resolve(candidate: str, api) -> Resolution:
+    name = normalize(candidate)
+    if name.count("/") > 1:
+        return Resolution(name, frozenset({name}), name)
+    data = api.get_repo(name)
+    canonical = normalize(data["full_name"])
+    names = {name, canonical, *(normalize(n) for n in data.get("former_names", []))}
+    for key in ("parent", "source"):
+        if data.get(key):
+            names.add(normalize(data[key]["full_name"]))
+    network = normalize((data.get("source") or data.get("parent") or data)["full_name"])
+    return Resolution(canonical, frozenset(names), network)
+
+
+def canonicalize_used(used: UsedSet, api) -> None:
+    """Resolve old used names too: GitHub exposes redirects, not a complete rename history."""
+    cache = {}
+    for names in used.sources.values():
+        for name in sorted(names):
+            if name not in cache:
+                try:
+                    cache[name] = resolve(name, api)
+                except RepositoryNotFound:
+                    cache[name] = Resolution(name, frozenset({name}), name)
+            names.update(cache[name].names)
+
+
+def rejection(candidate: Resolution, used: UsedSet, selected: set[str]) -> str | None:
+    for label, names in used.sources.items():
+        if candidate.names & names:
+            return label
+    if candidate.network in selected:
+        return "fork_network"
+    selected.add(candidate.network)
+    return None
+
+
+class RepositoryNotFound(ValueError):
+    """A historical or synthetic used identity no longer resolves; retain it verbatim."""
+
+
+class GitHub:
+    """Coordinator-only GET client. Exceptions never disclose request URLs or tokens."""
+
+    def __init__(self, token: str):
+        if not token:
+            raise ValueError("missing_gh_token")
+        self.token = token
+
+    def get_repo(self, name: str) -> dict:
+        from urllib.error import HTTPError
+        from urllib.request import Request, urlopen
+
+        request = Request(
+            "https://api.github.com/repos/" + normalize(name),
+            headers={
+                "Authorization": "Bearer " + self.token,
+                "Accept": "application/vnd.github+json",
+            },
+        )
+        try:
+            with urlopen(request, timeout=30) as response:
+                return json.load(response)
+        except HTTPError as exc:
+            if exc.code == 404:
+                raise RepositoryNotFound("github_not_found") from None
+            raise ValueError("github_resolution_failed") from None
+        except Exception:
+            raise ValueError("github_resolution_failed") from None
+
+
+def verify_tests(root: Path) -> dict:
+    """The coordinator runs the guard as a black box; only exit codes and counts escape."""
+    import subprocess
+    import sys
+    import tempfile
+    from xml.etree import ElementTree
+
+    with tempfile.TemporaryDirectory() as directory:
+        report = Path(directory) / "tests.xml"
+        done = subprocess.run(
+            [
+                sys.executable,
+                "-m",
+                "pytest",
+                "-q",
+                "-p",
+                "no:cacheprovider",
+                "tests/test_unseen_eligibility.py",
+                "tests/test_unseen_ledger.py",
+                "tests/test_independent_population.py",
+                "--junitxml=" + str(report),
+            ],
+            cwd=root,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+        if done.returncode != 0:
+            raise ValueError("guard_or_tests_failed")
+        suites = ElementTree.parse(report).getroot().iter("testsuite")
+        counts = {key: 0 for key in ("tests", "failures", "errors", "skipped")}
+        for suite in suites:
+            for key in counts:
+                counts[key] += int(suite.attrib[key])
+        if not counts["tests"]:
+            raise ValueError("empty_test_run")
+        return {"guard": "green", "test_exit_code": done.returncode, "tests": counts}
+
+
+def main(argv: list[str] | None = None) -> int:
+    import os
+
+    from openultrasast import config
+    from openultrasast.plane import memory
+    from openultrasast.plane.reconciler import results_root
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path, default=Path.cwd())
+    parser.add_argument("--local-store", default=None)
+    parser.add_argument("--s3-store", default="s3://")
+    parser.add_argument("--pointer-manifest", type=Path, action="append", default=[])
+    parser.add_argument("--known-empty", choices=(*SOURCES, "memory_local", "memory_s3"), action="append", default=[])
+    parser.add_argument("--output", type=Path)
+    parser.add_argument("--verify-tests", action="store_true")
+    args = parser.parse_args(argv)
+    step = "open_stores"
+    try:
+        config.load_dotenv(args.root / ".env")
+        stores = {
+            "local": memory.open_store(args.local_store or "file://" + str(results_root() / "memory"), read_only=True),
+            "s3": memory.open_store(args.s3_store, read_only=True),
+        }
+        step = "build_used"
+        used = build_used(args.root, stores=stores, pointer_manifests=args.pointer_manifest)
+        step = "report"
+        used.report(set(args.known_empty))  # prove inputs before API work
+        step = "canonicalize"
+        canonicalize_used(used, GitHub(os.environ.get("GH_TOKEN", "")))
+        step = "report"
+        report = used.report(set(args.known_empty))
+        if args.verify_tests:
+            step = "verify_tests"
+            report.update(verify_tests(args.root))
+        step = "report"
+        text = json.dumps(report, sort_keys=True, indent=2) + "\n"
+        if args.output:
+            args.output.parent.mkdir(parents=True, exist_ok=True)
+            with args.output.open("x") as handle:
+                handle.write(text)
+        print(text, end="")
+        return 0
+    except Exception as exc:
+        # Source paths, store exceptions and JSON parser errors can contain repository names.
+        report = {"status": "instrument_failure", "step": step, "type": type(exc).__name__}
+        if str(exc) in {"empty_source:" + label for label in (*SOURCES, "memory_local", "memory_s3")}:
+            report["source"] = str(exc).partition(":")[2]
+        print(json.dumps(report))
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())