search-with-proof: executor smoke driver and pilot selection (6.1)

20 development pairs, one per repository, 4 per language, 5 per family; identities
in a gitignored manifest, counts and digest committed. Executor smoke drives one
executor task end to end without a model and fails loudly on an unread clone.
kube-ax operator constraints as steering.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

benchmarks/ax/batch.py

@@ -561,7 +561,9 @@ class DockerLane(AXLane):
             ]
             for env in manifest["spec"]["env"]:
                 command += ["-e", env["name"] + "=" + env["value"]]
-            command += [manifest["spec"]["image"], *manifest["spec"]["command"][1:], "--heap-profile", "vm"]
+            command += [manifest["spec"]["image"], *manifest["spec"]["command"][1:]]
+            if "openultrasast.search.executor" not in manifest["spec"]["command"]:
+                command += ["--heap-profile", "vm"]
         elif args[0] == "get":
             command = ["docker", "inspect", "--format", "{{.State.Running}} {{.State.ExitCode}}", self._docker_name]
         else:
@@ -599,6 +601,12 @@ class SearchExecutorTask:
     def __init__(self, lane, checkout_archive, *, deadline=900):
         if not 0 < deadline <= 3600 or len(checkout_archive) > 128 * 1024**2:
             raise ValueError("executor task input limit")
+        if isinstance(lane, DockerLane):
+            import copy
+
+            lane = copy.copy(lane)
+            lane._diagnostics = []
+            lane.ax = lane._docker_command
         self.lane, self.archive, self.deadline = lane, checkout_archive, deadline
         token = uuid.uuid4().hex[:20]
         self.name = "ousast-engine-search-exec-" + token
@@ -610,9 +618,10 @@ class SearchExecutorTask:
         from openultrasast.search.executor import ObjectStoreExecutor
 
         self.end = time.monotonic() + self.deadline
-        if not AX_SLOTS.acquire(timeout=self.deadline):
-            raise CapacityDeadline
-        self.slot = True
+        if not isinstance(self.lane, DockerLane):
+            if not AX_SLOTS.acquire(timeout=self.deadline):
+                raise CapacityDeadline
+            self.slot = True
         expires = timedelta(seconds=self.deadline + 600)
         store = self.lane.store
         try:

benchmarks/measurements/2026-10-05-search-pilot-selection/record.json

@@ -0,0 +1,74 @@
+{
+  "biases": [
+    "known_vulnerable_location",
+    "labelled_family",
+    "runnable_project_preference"
+  ],
+  "catalogs": {
+    "bytes": 904050,
+    "files": 10,
+    "rows": 873
+  },
+  "eligible_pairs": 192,
+  "eligible_repositories": 192,
+  "exclusions": {
+    "bytes": 0,
+    "count": 0,
+    "files": 0,
+    "ledger_rows": 0,
+    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+  },
+  "gap_to_target": 0,
+  "guard": {
+    "guard": "green",
+    "test_exit_code": 0,
+    "tests": {
+      "errors": 0,
+      "failures": 0,
+      "skipped": 1,
+      "tests": 41
+    }
+  },
+  "language_floor_gaps": {
+    "java": 0,
+    "javascript": 0,
+    "php": 0,
+    "python": 0,
+    "typescript": 0
+  },
+  "local_metadata": {
+    "config_bytes": 0,
+    "manifest_bytes": 0,
+    "matched_checkouts": 0
+  },
+  "manifest_sha256": "b593d2760f525c38b205fb33866c383fead4666102a84a805150b32c55ceeb62",
+  "per_family": {
+    "injection": 5,
+    "output_encoding": 5,
+    "path": 5,
+    "untrusted_destination": 5
+  },
+  "per_language": {
+    "java": 4,
+    "javascript": 4,
+    "php": 4,
+    "python": 4,
+    "typescript": 4
+  },
+  "purpose": "feasibility_only",
+  "qualification": "not_an_unseen_evaluation_or_block_admission",
+  "rejected": {
+    "commits": 60,
+    "family": 62,
+    "language": 100,
+    "location": 1,
+    "not_development_fix": 453,
+    "repository": 5
+  },
+  "seed": 20261005,
+  "selected": 20,
+  "status": "selected",
+  "target": 20,
+  "unknown_manifest": 20,
+  "unknown_test_suite": 20
+}

benchmarks/search/README.md

@@ -0,0 +1,57 @@
+# Search feasibility tools
+
+These host-side tools make no model calls. Run them from the repository root.
+
+## Executor smoke
+
+```sh
+python -m benchmarks.search.executor_smoke \
+  --lane ax --image REGISTRY/IMAGE@sha256:DIGEST \
+  --repo-url https://github.com/OWNER/REPOSITORY --commit FULL_SHA \
+  --out /tmp/executor-smoke.json
+```
+
+`--lane docker` uses the same object-store mailbox and an already available local
+image (`--pull never`). Both lanes need the existing batch transport's S3 store
+configuration; credentials stay on the host. Running this command contacts the
+store and task runner; the task fetches a shallow HTTPS checkout and installs
+dependencies. Offline unit tests replace those network operations with a fake lane.
+
+One managed executor receives clone, list, read, grep, install, trivial run, and
+stop commands. Acquisition uses `git init` plus a depth-one fetch of the exact
+commit, retrying initial egress failures for up to 60 seconds. Install recipes are
+fixed and chosen from root pip/npm/composer/Maven manifests. Caches, temporary
+files, and pip products live under `/workspace`. A repository without a recognised
+manifest produces an instrument failure. If no README exists, the smoke reads
+another listed file and records its size to establish that input was opened.
+
+The JSON record holds per-command executor and round-trip wall times, exit codes
+(null for file tools), observed output sizes, and sampled allocated workspace
+peak, including unlinked command logs. Samples are not an instantaneous disk
+quota. Output text and presigned URLs are omitted. Zero clone files, a successful
+install with no output in under one second, command failure, missing runtime
+output, and cleanup errors produce `instrument_failure` and CLI exit 1. Context
+cleanup attempts task deletion and removal of mailbox objects on failure as well
+as success.
+
+## Pilot selection
+
+```sh
+python -m benchmarks.search.pilot_select --seed 20261005
+```
+
+Selection uses development catalog metadata only; it performs no downloads or
+model calls. It excludes unseen reservations and used sets, validates usage
+ledgers, and runs the existing reserved-repository guard as a black box. Additional
+local unseen metadata/ledgers can be supplied with `--exclude` and `--ledger`.
+Use `--cache` to inspect matching local checkouts for manifest/test-suite evidence;
+the default cache is `~/.cache/openultrasast/repos`. Cached metadata affects
+preference only and does not establish that the pinned revision builds.
+The selector never opens population v3 files itself.
+
+The default identity-bearing manifest is
+`benchmarks/search/private/pilot.json`, explicitly gitignored. The summary is
+`benchmarks/measurements/2026-10-05-search-pilot-selection/record.json`; it contains
+counts, selection biases, input instrumentation, and digests only. Neither output
+is overwritten. A successful selection is feasibility input, not evidence of
+buildability, oracle operation, unseen performance, or BLOCK eligibility.

benchmarks/search/executor_smoke.py

@@ -0,0 +1,184 @@
+"""One no-model executor task; run as python -m benchmarks.search.executor_smoke.
+
+Runtime acquisition/install needs network inside the executor. Unit tests are offline.
+Records contain measurements, never command output or presigned bearer URLs.
+"""
+
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import re
+import tarfile
+import time
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from benchmarks.ax.batch import AXLane, DockerLane, SearchExecutorTask, Workload, validate_image
+
+# URL and pin are positional shell arguments, never interpolated into shell code.
+# Retry the initial acquisition for delayed task egress activation (at most 60 s).
+CLONE = """git init -q .
+git remote add origin "$1"
+end=$(($(date +%s) + 60))
+until git -c protocol.version=2 fetch --depth=1 origin "$2"; do
+  test "$(date +%s)" -lt "$end" || exit 1
+  sleep 3
+done
+git checkout -q --detach FETCH_HEAD
+test "$(git rev-parse HEAD)" = "$2"
+"""
+
+
+def install_recipe(files):
+    """Fixed recipes, selected from checkout root manifests only."""
+    files = set(files)
+    if "requirements.txt" in files:
+        return "pip", ["python3", "-m", "pip", "install", "--target", "/workspace/packages", "-r", "requirements.txt"]
+    if files & {"pyproject.toml", "setup.py", "setup.cfg"}:
+        return "pip", ["python3", "-m", "pip", "install", "--target", "/workspace/packages", "."]
+    if "package.json" in files:
+        return "npm", ["npm", "ci" if "package-lock.json" in files else "install"]
+    if "composer.json" in files:
+        return "composer", ["composer", "install", "--no-interaction", "--prefer-dist"]
+    if "pom.xml" in files:
+        return "maven", ["mvn", "-B", "-Dmaven.repo.local=/workspace/.m2", "-DskipTests", "package"]
+    raise ValueError("no recognised install manifest")
+
+
+def run_smoke(lane, repo_url, commit, *, deadline=900):
+    record = {
+        "status": "instrument_failure",
+        "commands": [],
+        "scratch_peak_bytes": 0,
+        "scratch_measurement": "sampled workspace allocated bytes plus command logs",
+        "model_calls": 0,
+    }
+    started = time.monotonic()
+    try:
+        parts = urlsplit(repo_url)
+        if parts.scheme != "https" or not parts.hostname or parts.username or parts.password or parts.query or parts.fragment:
+            raise ValueError("credential-free HTTPS repository URL required")
+        if not re.fullmatch("[0-9a-f]{40}", commit):
+            raise ValueError("full lowercase commit SHA required")
+        archive = io.BytesIO()
+        with tarfile.open(fileobj=archive, mode="w"):
+            pass
+        # The empty checkout is filled by the first command inside the key-free task.
+        with SearchExecutorTask(lane, archive.getvalue(), deadline=deadline) as task:
+            record["task"] = task.name
+
+            def command(step, name, args):
+                remaining = deadline - (time.monotonic() - started) - 2
+                if remaining <= 0:
+                    raise TimeoutError("smoke deadline")
+                timeout = min(300, remaining)
+                if name == "run":
+                    args = {**args, "timeout_seconds": timeout}
+                before = time.monotonic()
+                result = task.client.submit(
+                    name, args, timeout_seconds=timeout, limits={"memory_bytes": 2 * 1024**3, "disk_bytes": 2 * 1024**3}
+                )
+                output_bytes = sum(result.get(k + "_bytes", len(result.get(k, "").encode())) for k in ("stdout", "stderr"))
+                if name != "run":
+                    output_bytes = len(json.dumps(result).encode())
+                row = {
+                    "step": step,
+                    "command": name,
+                    "wall_seconds": result.get("wall_seconds", time.monotonic() - before),
+                    "roundtrip_seconds": time.monotonic() - before,
+                    "exit_code": result.get("exit_code"),
+                    "output_bytes": output_bytes,
+                    "scratch_peak_bytes": result.get("scratch_peak_bytes"),
+                    "truncated": result.get("truncated", False),
+                }
+                if "input_bytes" in result:
+                    row["input_bytes"] = result["input_bytes"]
+                record["commands"].append(row)
+                record["scratch_peak_bytes"] = max(record["scratch_peak_bytes"], result.get("scratch_peak_bytes", 0))
+                if result.get("error") or result.get("timed_out") or result.get("status") == "could_not_build":
+                    raise ValueError(step + " executor failure")
+                if name == "run" and result.get("exit_code") != 0:
+                    raise ValueError(step + " nonzero or missing exit code")
+                if step == "install" and output_bytes == 0 and row["wall_seconds"] < 1:
+                    raise ValueError("empty fast install: exit 0 with empty output in under 1 s")
+                return result
+
+            command("clone", "run", {"command": ["sh", "-ec", CLONE, "clone", repo_url, commit]})
+            files = command("list_files", "list_files", {}).get("files", [])
+            if not files:
+                raise ValueError("clone yielded zero files")
+            record["files_listed"] = len(files)
+            readme = next((p for p in files if Path(p).name.lower().startswith("readme")), None)
+            # Always open input even when the repository has no README.
+            read = command("read_file", "read_file", {"path": readme or files[0]})
+            record["readme_present"] = readme is not None
+            record["input_bytes"] = read.get("input_bytes")
+            if not isinstance(record["input_bytes"], int):
+                raise ValueError("file read lacks input size")
+            command("grep", "grep", {"pattern": "test"})
+            recipe, argv = install_recipe(files)
+            record["recipe"] = recipe
+            command("install", "run", {"command": argv})
+            trivial = ["node", "-e", "console.log(1)"] if recipe == "npm" else ["python3", "-c", "print(1)"]
+            ran = command("trivial", "run", {"command": trivial})
+            if ran.get("stdout", "").strip() != "1":
+                raise ValueError("trivial command did not print 1")
+            if not command("stop", "stop", {}).get("stopped"):
+                raise ValueError("stop not acknowledged")
+            record["status"] = "ok"
+    except Exception as exc:
+        record["status"] = "instrument_failure"
+        # Only our fixed diagnostic strings are safe; transport errors may contain URLs.
+        record["reason"] = (
+            str(exc)
+            if isinstance(exc, ValueError)
+            and str(exc)
+            in {
+                "credential-free HTTPS repository URL required",
+                "full lowercase commit SHA required",
+                "no recognised install manifest",
+                "clone yielded zero files",
+                "file read lacks input size",
+                "trivial command did not print 1",
+                "stop not acknowledged",
+                "empty fast install: exit 0 with empty output in under 1 s",
+                *(
+                    step + suffix
+                    for step in ("clone", "list_files", "read_file", "grep", "install", "trivial", "stop")
+                    for suffix in (" executor failure", " nonzero or missing exit code")
+                ),
+            }
+            else type(exc).__name__
+        )
+    record["wall_seconds"] = time.monotonic() - started
+    return record
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--lane", choices=("ax", "docker"), required=True)
+    parser.add_argument("--image", required=True)
+    parser.add_argument("--repo-url", required=True)
+    parser.add_argument("--commit", required=True)
+    parser.add_argument("--out", type=Path, required=True, help="One JSON record")
+    parser.add_argument("--deadline", type=float, default=900)
+    parser.add_argument("--ax-bin", default="ax")
+    parser.add_argument("--atespace", default="default")
+    parser.add_argument("--kubeconfig")
+    args = parser.parse_args(argv)
+    try:
+        validate_image(args.image)
+        lane = (DockerLane if args.lane == "docker" else AXLane)(args, Workload(args.image, frozenset(), args.deadline, None))
+        record = run_smoke(lane, args.repo_url, args.commit, deadline=args.deadline)
+    except Exception as exc:
+        record = {"status": "instrument_failure", "reason": "lane setup: " + type(exc).__name__}
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(record))
+    return int(record["status"] != "ok")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/search/pilot_select.py

@@ -0,0 +1,360 @@
+"""Offline development-pair feasibility pilot; identities stay in a gitignored manifest.
+
+Run from the repository root with python -m benchmarks.search.pilot_select.
+The existing reserved guard is executed as a black box: this module never reads v3.
+"""
+
+from __future__ import annotations
+
+import argparse
+import configparser
+import hashlib
+import json
+import os
+import re
+import subprocess
+from collections import Counter
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from benchmarks.unseen import eligibility, ledger
+from openultrasast.model.taxonomy import load_families
+
+LANGUAGES = ("python", "javascript", "typescript", "php", "java")
+FAMILIES = ("injection", "path", "output_encoding", "untrusted_destination")
+SHA = re.compile(r"[0-9a-f]{40}")
+MANIFESTS = {
+    "requirements.txt": "pip",
+    "pyproject.toml": "pip",
+    "setup.py": "pip",
+    "Pipfile": "pip",
+    "package.json": "npm",
+    "composer.json": "composer",
+    "pom.xml": "maven",
+    "build.gradle": "gradle",
+    "build.gradle.kts": "gradle",
+}
+
+
+def encoded(value) -> bytes:
+    return (json.dumps(value, sort_keys=True, indent=2) + "\n").encode()
+
+
+def safe(path: Path) -> bool:
+    return not eligibility.excluded(path) and not eligibility.excluded(path.resolve()) and not path.is_symlink()
+
+
+def load_catalogs(root: Path) -> tuple[list[dict], dict]:
+    """Read raw metadata only; PairCase materialization would unnecessarily open source excerpts."""
+    rows = []
+    counts = {"files": 0, "bytes": 0, "rows": 0}
+    directory = (root / "benchmarks/pairs").resolve()
+    for path in sorted(directory.rglob("catalog.toml")):
+        if not safe(path) or not path.resolve().is_relative_to(directory):
+            continue
+        raw = path.read_bytes()
+        items = eligibility.parse(path, raw).get("pair", [])
+        if not isinstance(items, list) or not all(isinstance(row, dict) for row in items):
+            raise ValueError("invalid_catalog")
+        rows.extend(items)
+        counts["files"] += 1
+        counts["bytes"] += len(raw)
+        counts["rows"] += len(items)
+    if not counts["bytes"] or not rows:
+        raise ValueError("empty_catalogs")
+    return rows, counts
+
+
+def exclusions(root: Path, extra: tuple[Path, ...] = (), ledgers: tuple[Path, ...] = ()) -> tuple[set[str], dict]:
+    """Unseen-local reservations/used sets, including private journals; never the development used set.
+
+    eligibility.build_used collects development data itself, so its union cannot exclude a
+    development pilot. The unseen usage ledger has opaque slices, not identities: every pool
+    is excluded in full regardless of usage, and ledger integrity is checked separately.
+    """
+    directory = root / "benchmarks/unseen"
+    paths = {
+        p
+        for p in directory.rglob("*")
+        if p.suffix in {".toml", ".json", ".jsonl", ".yaml", ".yml"} and "fixtures" not in p.relative_to(directory).parts
+    }
+    paths.update(extra)
+    ledger_paths = set(ledgers)
+    ledger_paths.update(p for p in paths if any(word in p.stem.lower() for word in ("ledger", "usage")))
+    paths.update(ledger_paths)
+    names: set[str] = set()
+    report = {"files": 0, "bytes": 0, "ledger_rows": 0}
+    for path in sorted(paths):
+        if not safe(path):
+            raise ValueError("unsafe_exclusion_source")
+        raw = path.read_bytes()
+        report["files"] += 1
+        report["bytes"] += len(raw)
+        if path in ledger_paths:
+            report["ledger_rows"] += len(ledger.read(path))
+        else:
+            value = eligibility.parse(path, raw)
+
+            # TOML [[repository]] is a collection, whereas repositories() treats
+            # a scalar "repository" field as an identity. Walk containers too.
+            def collect(item):
+                names.update(eligibility.repositories(item))
+                if isinstance(item, str) and eligibility.NAME.fullmatch(item):
+                    names.add(eligibility.normalize(item))
+                if isinstance(item, dict):
+                    for child in item.values():
+                        if isinstance(child, (dict, list)):
+                            collect(child)
+                elif isinstance(item, list):
+                    for child in item:
+                        collect(child)
+
+            collect(value)
+    report.update(count=len(names), sha256=hashlib.sha256("\n".join(sorted(names)).encode()).hexdigest())
+    return names, report
+
+
+def cache_safe(path: Path) -> bool:
+    # Git config is intentionally readable here; every other exclusion and all
+    # symlink ancestors remain forbidden before opening files or listing trees.
+    lexical = path.absolute()
+    resolved = path.resolve()
+    return not any("v3" in part.lower() or part in {"__pycache__", ".venv"} for p in (lexical, resolved) for part in p.parts) and not any(
+        p.is_symlink() for p in (lexical, *lexical.parents)
+    )
+
+
+def enrich(rows: list[dict], caches: tuple[Path, ...], *, excluded: set[str] | None = None) -> dict:
+    """Prefer metadata actually present in a matching local clone, without fetching.
+
+    This is only a preference: a cached checkout is not proof the pinned revision builds.
+    Only development-candidate paths are opened; unrelated cached repositories are not read.
+    """
+    report = {"matched_checkouts": 0, "config_bytes": 0, "manifest_bytes": 0}
+    seen = {}
+    for row in rows:
+        try:
+            name = eligibility.normalize(row.get("repo", ""))
+        except (ValueError, TypeError):
+            continue
+        if name in (excluded or set()):
+            continue
+        if name not in seen:
+            seen[name] = []
+            for cache in caches:
+                for checkout in (cache / name, cache / name.rsplit("/", 1)[-1]):
+                    config = checkout / ".git/config"
+                    if not cache_safe(config) or not config.is_file():
+                        continue
+                    parser = configparser.ConfigParser()
+                    raw = config.read_bytes()
+                    parser.read_string(raw.decode())
+                    origin = parser.get('remote "origin"', "url", fallback="")
+                    try:
+                        matching = eligibility.normalize(origin) == name
+                    except ValueError:
+                        matching = False
+                    if not matching:
+                        continue
+                    files = []
+                    for manifest in MANIFESTS:
+                        path = checkout / manifest
+                        if cache_safe(path) and path.is_file():
+                            report["manifest_bytes"] += len(path.read_bytes())
+                            files.append(manifest)
+                    for folder in ("tests", "test", "spec", "__tests__", "src/test"):
+                        directory = checkout / folder
+                        if cache_safe(directory) and directory.is_dir() and any(directory.iterdir()):
+                            files.append(folder + "/")
+                    seen[name] = files
+                    report["matched_checkouts"] += 1
+                    report["config_bytes"] += len(raw)
+                    break
+                if seen[name]:
+                    break
+        row["files"] = sorted(set(row.get("files", []) + seen[name]))
+    return report
+
+
+def candidates(rows: list[dict], excluded: set[str]) -> tuple[list[dict], dict]:
+    taxonomy = load_families()
+    used = eligibility.UsedSet(sources={"unseen": excluded})
+    out, rejected = [], Counter()
+    for row in rows:
+        reason = None
+        try:
+            raw_repo = row.get("repo", "")
+            repo = eligibility.normalize(raw_repo)
+            if "://" in raw_repo and not raw_repo.startswith("https://"):
+                raise ValueError
+            url = raw_repo if raw_repo.startswith("https://") else "https://github.com/" + repo
+            if repo.count("/") != 1:
+                raise ValueError
+            parsed = urlsplit(url)
+            if (
+                parsed.netloc != "github.com"
+                or parsed.query
+                or parsed.fragment
+                or parsed.path.strip("/").removesuffix(".git").lower() != repo
+            ):
+                raise ValueError
+            if row.get("fix_repo") and eligibility.normalize(row["fix_repo"]) != repo:
+                raise ValueError
+        except (ValueError, TypeError):
+            rejected["repository"] += 1
+            continue
+        if row.get("split", "train") != "train" or row.get("provenance") == "synthetic":
+            reason = "not_development_fix"
+        elif row.get("language") not in LANGUAGES:
+            reason = "language"
+        elif eligibility.rejection(eligibility.Resolution(repo, frozenset({repo}), repo), used, set()):
+            reason = "unseen"
+        vulnerable, fixed = row.get("parent", ""), row.get("commit", "")
+        if not all(isinstance(s, str) and SHA.fullmatch(s) for s in (vulnerable, fixed)) or vulnerable == fixed:
+            reason = reason or "commits"
+        expected = row.get("expected", [])
+        families = {row["family"]} if row.get("family") else set()
+        for item in [row, *expected]:
+            family = taxonomy.family_of_cwe(item.get("cwe", ""))
+            if family:
+                families.add(family.id)
+        families &= set(FAMILIES)
+        function = row.get("function") or next((r.get("function") for r in expected if r.get("function")), "")
+        file = row.get("path") or row.get("relpath", "")
+        if len(families) != 1:
+            reason = reason or "family"
+        if not function or function in {"<anon>", "<module>", "*"} or not file:
+            reason = reason or "location"
+        if reason:
+            rejected[reason] += 1
+            continue
+        files = row.get("files", [])
+        manifest = next((MANIFESTS[Path(f).name] for f in sorted(files) if Path(f).name in MANIFESTS), None)
+        tests = True if any(re.search(r"(^|/)(tests?|__tests__|spec)(/|$)|(^|/)test_[^/]+", f) for f in files) else None
+        out.append(
+            {
+                "repo": url.removesuffix(".git").rstrip("/"),
+                "vulnerable": vulnerable,
+                "fixed": fixed,
+                "family": next(iter(families)),
+                "language": row["language"],
+                "file": file,
+                "function": function,
+                "manifest": manifest,
+                "has_tests": tests,
+            }
+        )
+    return out, dict(sorted(rejected.items()))
+
+
+def select(rows: list[dict], *, seed: int, count: int = 20) -> list[dict]:
+    """Balance language/family representation, then prefer evidenced runnable projects.
+
+    Seeded SHA ordering is invariant to catalog ordering and Python hash randomization.
+    At most one pair per normalized repository is selected.
+    """
+    if count < 1:
+        raise ValueError("invalid_count")
+    remaining = sorted(rows, key=lambda row: hashlib.sha256(str(seed).encode() + encoded(row)).digest())
+    selected, used = [], set()
+    languages, families = Counter(), Counter()
+    while remaining and len(selected) < count:
+        row = min(
+            remaining,
+            key=lambda r: (
+                languages[r["language"]] >= 3,
+                languages[r["language"]],
+                families[r["family"]],
+                not (r["has_tests"] and r["manifest"]),
+                not r["manifest"],
+                not r["has_tests"],
+            ),
+        )
+        selected.append(row)
+        used.add(eligibility.normalize(row["repo"]))
+        languages[row["language"]] += 1
+        families[row["family"]] += 1
+        remaining = [r for r in remaining if eligibility.normalize(r["repo"]) not in used]
+    return selected
+
+
+def summary(rows: list[dict], *, seed: int, manifest: bytes, target: int = 20) -> dict:
+    languages = Counter(row["language"] for row in rows)
+    return {
+        "status": "selected" if len(rows) == target else "insufficient_candidates",
+        "purpose": "feasibility_only",
+        "seed": seed,
+        "target": target,
+        "selected": len(rows),
+        "gap_to_target": max(0, target - len(rows)),
+        "per_language": {lang: languages[lang] for lang in LANGUAGES},
+        "per_family": {family: sum(r["family"] == family for r in rows) for family in FAMILIES},
+        "language_floor_gaps": {lang: max(0, 3 - languages[lang]) for lang in LANGUAGES},
+        "unknown_test_suite": sum(r["has_tests"] is None for r in rows),
+        "unknown_manifest": sum(r["manifest"] is None for r in rows),
+        "manifest_sha256": hashlib.sha256(manifest).hexdigest(),
+        "biases": ["known_vulnerable_location", "labelled_family", "runnable_project_preference"],
+        "qualification": "not_an_unseen_evaluation_or_block_admission",
+    }
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path, default=Path.cwd())
+    parser.add_argument("--seed", type=int, default=20261005)
+    parser.add_argument("--count", type=int, default=20)
+    parser.add_argument("--exclude", type=Path, action="append", default=[])
+    parser.add_argument("--ledger", type=Path, action="append", default=[])
+    parser.add_argument("--cache", type=Path, action="append", default=[])
+    parser.add_argument("--manifest", type=Path, default=Path("benchmarks/search/private/pilot.json"))
+    parser.add_argument("--output", type=Path, default=Path("benchmarks/measurements/2026-10-05-search-pilot-selection/record.json"))
+    args = parser.parse_args(argv)
+    private = args.root / args.manifest
+    created = False
+    try:
+        if not private.resolve().is_relative_to((args.root / "benchmarks/search/private").resolve()):
+            raise ValueError("manifest_must_be_private")
+        ignored = subprocess.run(["git", "check-ignore", "-q", str(private)], cwd=args.root, capture_output=True)
+        if ignored.returncode != 0:
+            raise ValueError("manifest_not_gitignored")
+        guard = eligibility.verify_tests(args.root)
+        rows, instrument = load_catalogs(args.root)
+        excluded, exclusion_report = exclusions(args.root, tuple(args.exclude), tuple(args.ledger))
+        # The black-box guard has already approved the development catalogs.
+        metadata = enrich(rows, tuple(args.cache) or (Path.home() / ".cache/openultrasast/repos",), excluded=excluded)
+        available, rejected = candidates(rows, excluded)
+        selected = select(available, seed=args.seed, count=args.count)
+        payload = encoded({"seed": args.seed, "pairs": selected})
+        private.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        with private.open("xb") as handle:
+            created = True
+            os.chmod(private, 0o600)
+            handle.write(payload)
+        # The URL-bearing private selection now lies in the guard's development tree,
+        # so even catalog bare-name aliases are checked without reading v3 ourselves.
+        guard = eligibility.verify_tests(args.root)
+        record = summary(selected, seed=args.seed, manifest=payload, target=args.count)
+        record.update(
+            catalogs=instrument,
+            exclusions=exclusion_report,
+            rejected=rejected,
+            local_metadata=metadata,
+            eligible_pairs=len(available),
+            eligible_repositories=len({r["repo"] for r in available}),
+            guard=guard,
+        )
+        output = args.root / args.output
+        output.parent.mkdir(parents=True, exist_ok=True)
+        with output.open("xb") as handle:
+            handle.write(encoded(record))
+        print(encoded(record).decode(), end="")
+        return 0 if len(selected) == args.count else 2
+    except Exception as exc:
+        if created:
+            private.unlink(missing_ok=True)
+        print(json.dumps({"status": "instrument_failure", "reason": type(exc).__name__}))
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())