search-with-proof: executor smoke driver and pilot selection (6.1)

20 development pairs, one per repository, 4 per language, 5 per family; identities
in a gitignored manifest, counts and digest committed. Executor smoke drives one
executor task end to end without a model and fails loudly on an unread clone.
kube-ax operator constraints as steering.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

benchmarks/search/executor_smoke.py

@@ -0,0 +1,184 @@
+"""One no-model executor task; run as python -m benchmarks.search.executor_smoke.
+
+Runtime acquisition/install needs network inside the executor. Unit tests are offline.
+Records contain measurements, never command output or presigned bearer URLs.
+"""
+
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import re
+import tarfile
+import time
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from benchmarks.ax.batch import AXLane, DockerLane, SearchExecutorTask, Workload, validate_image
+
+# URL and pin are positional shell arguments, never interpolated into shell code.
+# Retry the initial acquisition for delayed task egress activation (at most 60 s).
+CLONE = """git init -q .
+git remote add origin "$1"
+end=$(($(date +%s) + 60))
+until git -c protocol.version=2 fetch --depth=1 origin "$2"; do
+  test "$(date +%s)" -lt "$end" || exit 1
+  sleep 3
+done
+git checkout -q --detach FETCH_HEAD
+test "$(git rev-parse HEAD)" = "$2"
+"""
+
+
+def install_recipe(files):
+    """Fixed recipes, selected from checkout root manifests only."""
+    files = set(files)
+    if "requirements.txt" in files:
+        return "pip", ["python3", "-m", "pip", "install", "--target", "/workspace/packages", "-r", "requirements.txt"]
+    if files & {"pyproject.toml", "setup.py", "setup.cfg"}:
+        return "pip", ["python3", "-m", "pip", "install", "--target", "/workspace/packages", "."]
+    if "package.json" in files:
+        return "npm", ["npm", "ci" if "package-lock.json" in files else "install"]
+    if "composer.json" in files:
+        return "composer", ["composer", "install", "--no-interaction", "--prefer-dist"]
+    if "pom.xml" in files:
+        return "maven", ["mvn", "-B", "-Dmaven.repo.local=/workspace/.m2", "-DskipTests", "package"]
+    raise ValueError("no recognised install manifest")
+
+
+def run_smoke(lane, repo_url, commit, *, deadline=900):
+    record = {
+        "status": "instrument_failure",
+        "commands": [],
+        "scratch_peak_bytes": 0,
+        "scratch_measurement": "sampled workspace allocated bytes plus command logs",
+        "model_calls": 0,
+    }
+    started = time.monotonic()
+    try:
+        parts = urlsplit(repo_url)
+        if parts.scheme != "https" or not parts.hostname or parts.username or parts.password or parts.query or parts.fragment:
+            raise ValueError("credential-free HTTPS repository URL required")
+        if not re.fullmatch("[0-9a-f]{40}", commit):
+            raise ValueError("full lowercase commit SHA required")
+        archive = io.BytesIO()
+        with tarfile.open(fileobj=archive, mode="w"):
+            pass
+        # The empty checkout is filled by the first command inside the key-free task.
+        with SearchExecutorTask(lane, archive.getvalue(), deadline=deadline) as task:
+            record["task"] = task.name
+
+            def command(step, name, args):
+                remaining = deadline - (time.monotonic() - started) - 2
+                if remaining <= 0:
+                    raise TimeoutError("smoke deadline")
+                timeout = min(300, remaining)
+                if name == "run":
+                    args = {**args, "timeout_seconds": timeout}
+                before = time.monotonic()
+                result = task.client.submit(
+                    name, args, timeout_seconds=timeout, limits={"memory_bytes": 2 * 1024**3, "disk_bytes": 2 * 1024**3}
+                )
+                output_bytes = sum(result.get(k + "_bytes", len(result.get(k, "").encode())) for k in ("stdout", "stderr"))
+                if name != "run":
+                    output_bytes = len(json.dumps(result).encode())
+                row = {
+                    "step": step,
+                    "command": name,
+                    "wall_seconds": result.get("wall_seconds", time.monotonic() - before),
+                    "roundtrip_seconds": time.monotonic() - before,
+                    "exit_code": result.get("exit_code"),
+                    "output_bytes": output_bytes,
+                    "scratch_peak_bytes": result.get("scratch_peak_bytes"),
+                    "truncated": result.get("truncated", False),
+                }
+                if "input_bytes" in result:
+                    row["input_bytes"] = result["input_bytes"]
+                record["commands"].append(row)
+                record["scratch_peak_bytes"] = max(record["scratch_peak_bytes"], result.get("scratch_peak_bytes", 0))
+                if result.get("error") or result.get("timed_out") or result.get("status") == "could_not_build":
+                    raise ValueError(step + " executor failure")
+                if name == "run" and result.get("exit_code") != 0:
+                    raise ValueError(step + " nonzero or missing exit code")
+                if step == "install" and output_bytes == 0 and row["wall_seconds"] < 1:
+                    raise ValueError("empty fast install: exit 0 with empty output in under 1 s")
+                return result
+
+            command("clone", "run", {"command": ["sh", "-ec", CLONE, "clone", repo_url, commit]})
+            files = command("list_files", "list_files", {}).get("files", [])
+            if not files:
+                raise ValueError("clone yielded zero files")
+            record["files_listed"] = len(files)
+            readme = next((p for p in files if Path(p).name.lower().startswith("readme")), None)
+            # Always open input even when the repository has no README.
+            read = command("read_file", "read_file", {"path": readme or files[0]})
+            record["readme_present"] = readme is not None
+            record["input_bytes"] = read.get("input_bytes")
+            if not isinstance(record["input_bytes"], int):
+                raise ValueError("file read lacks input size")
+            command("grep", "grep", {"pattern": "test"})
+            recipe, argv = install_recipe(files)
+            record["recipe"] = recipe
+            command("install", "run", {"command": argv})
+            trivial = ["node", "-e", "console.log(1)"] if recipe == "npm" else ["python3", "-c", "print(1)"]
+            ran = command("trivial", "run", {"command": trivial})
+            if ran.get("stdout", "").strip() != "1":
+                raise ValueError("trivial command did not print 1")
+            if not command("stop", "stop", {}).get("stopped"):
+                raise ValueError("stop not acknowledged")
+            record["status"] = "ok"
+    except Exception as exc:
+        record["status"] = "instrument_failure"
+        # Only our fixed diagnostic strings are safe; transport errors may contain URLs.
+        record["reason"] = (
+            str(exc)
+            if isinstance(exc, ValueError)
+            and str(exc)
+            in {
+                "credential-free HTTPS repository URL required",
+                "full lowercase commit SHA required",
+                "no recognised install manifest",
+                "clone yielded zero files",
+                "file read lacks input size",
+                "trivial command did not print 1",
+                "stop not acknowledged",
+                "empty fast install: exit 0 with empty output in under 1 s",
+                *(
+                    step + suffix
+                    for step in ("clone", "list_files", "read_file", "grep", "install", "trivial", "stop")
+                    for suffix in (" executor failure", " nonzero or missing exit code")
+                ),
+            }
+            else type(exc).__name__
+        )
+    record["wall_seconds"] = time.monotonic() - started
+    return record
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--lane", choices=("ax", "docker"), required=True)
+    parser.add_argument("--image", required=True)
+    parser.add_argument("--repo-url", required=True)
+    parser.add_argument("--commit", required=True)
+    parser.add_argument("--out", type=Path, required=True, help="One JSON record")
+    parser.add_argument("--deadline", type=float, default=900)
+    parser.add_argument("--ax-bin", default="ax")
+    parser.add_argument("--atespace", default="default")
+    parser.add_argument("--kubeconfig")
+    args = parser.parse_args(argv)
+    try:
+        validate_image(args.image)
+        lane = (DockerLane if args.lane == "docker" else AXLane)(args, Workload(args.image, frozenset(), args.deadline, None))
+        record = run_smoke(lane, args.repo_url, args.commit, deadline=args.deadline)
+    except Exception as exc:
+        record = {"status": "instrument_failure", "reason": "lane setup: " + type(exc).__name__}
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n")
+    print(json.dumps(record))
+    return int(record["status"] != "ok")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())