diff --git a/.gitignore b/.gitignore index 82df916..dca4b01 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ site/ # Unseen evaluation pointers for non-permissive repositories. benchmarks/unseen/private/ + +# Feasibility pilot identities remain local; only aggregate records are tracked. +benchmarks/search/private/ diff --git a/.kiro/specs/search-with-proof/tasks.md b/.kiro/specs/search-with-proof/tasks.md index a7035f5..8af56de 100644 --- a/.kiro/specs/search-with-proof/tasks.md +++ b/.kiro/specs/search-with-proof/tasks.md @@ -46,7 +46,7 @@ gates, runs every networked or paid step. - [ ] 5.3 Smoke test with a stub model (no spend) end to end on one toy pair on kube-ax. - [ ] 6. Feasibility pilot (paid, after go-ahead) - - [ ] 6.1 Select 20 development-corpus pairs (never v3 or the unseen pool) in the four pool families with an owned + - [x] 6.1 Select 20 development-corpus pairs (never v3 or the unseen pool) in the four pool families with an owned oracle, across the five languages; record selection advantages. _Requirements: 6.1_ - [ ] 6.2 Run 3 searches, measure real cost per search, ask the maintainer for the pilot ceiling. _Requirements: 4.3_ - [ ] 6.3 Run the pilot: both sides per pair, fixed side blind to the vulnerable revision; record demonstrated @@ -97,3 +97,19 @@ gates, runs every networked or paid step. live-cluster criteria; their code/local smoke is implemented. No network, paid calls, image builds, pushes or commits. Offline wheel build could not resolve the uncached uv-build backend, so installed-wheel and real AX/gVisor/kind/Docker image validation remain gaps. + +- Task 6.1 and executor smoke (2026-10-05, explicitly authorized): added + `benchmarks/search/executor_smoke.py` and `pilot_select.py`, with offline tests and usage documentation. + The smoke drives one managed, key-free executor mailbox and records per-command timing/output measurements + and sampled workspace peak; AX and Docker lifecycle cleanup is covered by fake-lane tests. No live task ran. + Pilot selection read 10 development catalogs (873 rows, 904,050 bytes), retained 192 eligible pairs and selected + 20 distinct repositories: four per language and five per family. Identities live only in the gitignored local + manifest; the aggregate record is `benchmarks/measurements/2026-10-05-search-pilot-selection/record.json`. + All 20 selections have unknown manifest/test-suite evidence; no matching local cache or frozen unseen-pool + metadata was present. These are feasibility inputs, not proof of buildability or qualification. + Both task-local independent reviews approved after a reserved-cache-symlink regression was fixed. + Fresh requested search/AX/plane tests, module audit, independent/reserved and unseen guards, and label guards: + 287 passed, four skipped, exit 0. Skips: Chromium isolation, two localhost-socket cases, and no frozen pool. + Ruff and changed-source mypy pass; audit: 166 load-bearing, six standalone, zero orphaned. + No network or commits. The requested main merge could not update read-only Git metadata; HEAD and main + had identical file trees. Live AX/Docker executor smoke and oracle probes remain unmeasured. diff --git a/.kiro/steering/kube-ax.md b/.kiro/steering/kube-ax.md new file mode 100644 index 0000000..20d7447 --- /dev/null +++ b/.kiro/steering/kube-ax.md @@ -0,0 +1,30 @@ +# kube-ax operator constraints + +Verified by the operator on 2026-10-05. These constraints apply across all workloads. + +- Two workers, one task per worker, at most two tasks in flight cluster-wide. AX has no + queue: a third apply returns `no free workers available`. Treat this as back-pressure, + retry the same task/lane with 15–30 seconds of jitter until its deadline, and never + count it as a sandbox failure or trigger VM fallback. The host dispatcher shares a + two-slot reservation across workload instances and long-lived search executors. +- `/` and `/tmp` are RAM-backed and consume the worker's 3 GiB. `/workspace` is the only + disk-backed path, with about 2 GiB usable per task. Executor and verifier entrypoints + use `/workspace/tmp`, `/workspace/.npm`, `/workspace/.m2`, `/workspace/.composer`, + `/workspace/.pip`, and `/workspace/.gradle` through the respective tool environment + variables. Builds and extracted checkouts live under `/workspace` too. The default + aggregate workspace scratch guard is 2 GiB; `OUSAST_SCRATCH_BYTES` configures it. + Exceeding it returns `could_not_build`, reason `scratch limit`. The guard polls + during child execution and checks after exit; it is not an instantaneous filesystem + quota or a replacement for the worker memory limit. +- `ousast-engine-search-verify-*` tasks have no egress. The requested exception for + `files.because-security.com:443` is **pending**, not operationally verified. They + receive one presigned archive containing `checkout/`, `products/`, and `spec.json`, + built/exported by an executor. Only `VERIFY_INPUT_URL` and `RESULT_URL` cross the + verifier boundary. Verifiers reject build recipes and do not install dependencies + or fetch repositories. Executors finish and release their slots before verifier + repetitions start. Both task types must be provisioned without model credentials. +- All tasks share one public IP. Repository acquisition uses HTTPS git clone from + `github.com` or archives from `codeload.github.com`, never `api.github.com`. Host-side + metadata queries are a separate concern. Search tasks receive checkout archives; + executor tool guidance carries the same acquisition rule. Retain the 60-second + first-download retry for delayed task egress policy activation. diff --git a/benchmarks/ax/batch.py b/benchmarks/ax/batch.py index 9a5801b..9f387d9 100644 --- a/benchmarks/ax/batch.py +++ b/benchmarks/ax/batch.py @@ -561,7 +561,9 @@ class DockerLane(AXLane): ] for env in manifest["spec"]["env"]: command += ["-e", env["name"] + "=" + env["value"]] - command += [manifest["spec"]["image"], *manifest["spec"]["command"][1:], "--heap-profile", "vm"] + command += [manifest["spec"]["image"], *manifest["spec"]["command"][1:]] + if "openultrasast.search.executor" not in manifest["spec"]["command"]: + command += ["--heap-profile", "vm"] elif args[0] == "get": command = ["docker", "inspect", "--format", "{{.State.Running}} {{.State.ExitCode}}", self._docker_name] else: @@ -599,6 +601,12 @@ class SearchExecutorTask: def __init__(self, lane, checkout_archive, *, deadline=900): if not 0 < deadline <= 3600 or len(checkout_archive) > 128 * 1024**2: raise ValueError("executor task input limit") + if isinstance(lane, DockerLane): + import copy + + lane = copy.copy(lane) + lane._diagnostics = [] + lane.ax = lane._docker_command self.lane, self.archive, self.deadline = lane, checkout_archive, deadline token = uuid.uuid4().hex[:20] self.name = "ousast-engine-search-exec-" + token @@ -610,9 +618,10 @@ class SearchExecutorTask: from openultrasast.search.executor import ObjectStoreExecutor self.end = time.monotonic() + self.deadline - if not AX_SLOTS.acquire(timeout=self.deadline): - raise CapacityDeadline - self.slot = True + if not isinstance(self.lane, DockerLane): + if not AX_SLOTS.acquire(timeout=self.deadline): + raise CapacityDeadline + self.slot = True expires = timedelta(seconds=self.deadline + 600) store = self.lane.store try: diff --git a/benchmarks/measurements/2026-10-05-search-pilot-selection/record.json b/benchmarks/measurements/2026-10-05-search-pilot-selection/record.json new file mode 100644 index 0000000..b2c8f56 --- /dev/null +++ b/benchmarks/measurements/2026-10-05-search-pilot-selection/record.json @@ -0,0 +1,74 @@ +{ + "biases": [ + "known_vulnerable_location", + "labelled_family", + "runnable_project_preference" + ], + "catalogs": { + "bytes": 904050, + "files": 10, + "rows": 873 + }, + "eligible_pairs": 192, + "eligible_repositories": 192, + "exclusions": { + "bytes": 0, + "count": 0, + "files": 0, + "ledger_rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "gap_to_target": 0, + "guard": { + "guard": "green", + "test_exit_code": 0, + "tests": { + "errors": 0, + "failures": 0, + "skipped": 1, + "tests": 41 + } + }, + "language_floor_gaps": { + "java": 0, + "javascript": 0, + "php": 0, + "python": 0, + "typescript": 0 + }, + "local_metadata": { + "config_bytes": 0, + "manifest_bytes": 0, + "matched_checkouts": 0 + }, + "manifest_sha256": "b593d2760f525c38b205fb33866c383fead4666102a84a805150b32c55ceeb62", + "per_family": { + "injection": 5, + "output_encoding": 5, + "path": 5, + "untrusted_destination": 5 + }, + "per_language": { + "java": 4, + "javascript": 4, + "php": 4, + "python": 4, + "typescript": 4 + }, + "purpose": "feasibility_only", + "qualification": "not_an_unseen_evaluation_or_block_admission", + "rejected": { + "commits": 60, + "family": 62, + "language": 100, + "location": 1, + "not_development_fix": 453, + "repository": 5 + }, + "seed": 20261005, + "selected": 20, + "status": "selected", + "target": 20, + "unknown_manifest": 20, + "unknown_test_suite": 20 +} diff --git a/benchmarks/search/README.md b/benchmarks/search/README.md new file mode 100644 index 0000000..faf42d6 --- /dev/null +++ b/benchmarks/search/README.md @@ -0,0 +1,57 @@ +# Search feasibility tools + +These host-side tools make no model calls. Run them from the repository root. + +## Executor smoke + +```sh +python -m benchmarks.search.executor_smoke \ + --lane ax --image REGISTRY/IMAGE@sha256:DIGEST \ + --repo-url https://github.com/OWNER/REPOSITORY --commit FULL_SHA \ + --out /tmp/executor-smoke.json +``` + +`--lane docker` uses the same object-store mailbox and an already available local +image (`--pull never`). Both lanes need the existing batch transport's S3 store +configuration; credentials stay on the host. Running this command contacts the +store and task runner; the task fetches a shallow HTTPS checkout and installs +dependencies. Offline unit tests replace those network operations with a fake lane. + +One managed executor receives clone, list, read, grep, install, trivial run, and +stop commands. Acquisition uses `git init` plus a depth-one fetch of the exact +commit, retrying initial egress failures for up to 60 seconds. Install recipes are +fixed and chosen from root pip/npm/composer/Maven manifests. Caches, temporary +files, and pip products live under `/workspace`. A repository without a recognised +manifest produces an instrument failure. If no README exists, the smoke reads +another listed file and records its size to establish that input was opened. + +The JSON record holds per-command executor and round-trip wall times, exit codes +(null for file tools), observed output sizes, and sampled allocated workspace +peak, including unlinked command logs. Samples are not an instantaneous disk +quota. Output text and presigned URLs are omitted. Zero clone files, a successful +install with no output in under one second, command failure, missing runtime +output, and cleanup errors produce `instrument_failure` and CLI exit 1. Context +cleanup attempts task deletion and removal of mailbox objects on failure as well +as success. + +## Pilot selection + +```sh +python -m benchmarks.search.pilot_select --seed 20261005 +``` + +Selection uses development catalog metadata only; it performs no downloads or +model calls. It excludes unseen reservations and used sets, validates usage +ledgers, and runs the existing reserved-repository guard as a black box. Additional +local unseen metadata/ledgers can be supplied with `--exclude` and `--ledger`. +Use `--cache` to inspect matching local checkouts for manifest/test-suite evidence; +the default cache is `~/.cache/openultrasast/repos`. Cached metadata affects +preference only and does not establish that the pinned revision builds. +The selector never opens population v3 files itself. + +The default identity-bearing manifest is +`benchmarks/search/private/pilot.json`, explicitly gitignored. The summary is +`benchmarks/measurements/2026-10-05-search-pilot-selection/record.json`; it contains +counts, selection biases, input instrumentation, and digests only. Neither output +is overwritten. A successful selection is feasibility input, not evidence of +buildability, oracle operation, unseen performance, or BLOCK eligibility. diff --git a/benchmarks/search/executor_smoke.py b/benchmarks/search/executor_smoke.py new file mode 100644 index 0000000..0a886ff --- /dev/null +++ b/benchmarks/search/executor_smoke.py @@ -0,0 +1,184 @@ +"""One no-model executor task; run as python -m benchmarks.search.executor_smoke. + +Runtime acquisition/install needs network inside the executor. Unit tests are offline. +Records contain measurements, never command output or presigned bearer URLs. +""" + +from __future__ import annotations + +import argparse +import io +import json +import re +import tarfile +import time +from pathlib import Path +from urllib.parse import urlsplit + +from benchmarks.ax.batch import AXLane, DockerLane, SearchExecutorTask, Workload, validate_image + +# URL and pin are positional shell arguments, never interpolated into shell code. +# Retry the initial acquisition for delayed task egress activation (at most 60 s). +CLONE = """git init -q . +git remote add origin "$1" +end=$(($(date +%s) + 60)) +until git -c protocol.version=2 fetch --depth=1 origin "$2"; do + test "$(date +%s)" -lt "$end" || exit 1 + sleep 3 +done +git checkout -q --detach FETCH_HEAD +test "$(git rev-parse HEAD)" = "$2" +""" + + +def install_recipe(files): + """Fixed recipes, selected from checkout root manifests only.""" + files = set(files) + if "requirements.txt" in files: + return "pip", ["python3", "-m", "pip", "install", "--target", "/workspace/packages", "-r", "requirements.txt"] + if files & {"pyproject.toml", "setup.py", "setup.cfg"}: + return "pip", ["python3", "-m", "pip", "install", "--target", "/workspace/packages", "."] + if "package.json" in files: + return "npm", ["npm", "ci" if "package-lock.json" in files else "install"] + if "composer.json" in files: + return "composer", ["composer", "install", "--no-interaction", "--prefer-dist"] + if "pom.xml" in files: + return "maven", ["mvn", "-B", "-Dmaven.repo.local=/workspace/.m2", "-DskipTests", "package"] + raise ValueError("no recognised install manifest") + + +def run_smoke(lane, repo_url, commit, *, deadline=900): + record = { + "status": "instrument_failure", + "commands": [], + "scratch_peak_bytes": 0, + "scratch_measurement": "sampled workspace allocated bytes plus command logs", + "model_calls": 0, + } + started = time.monotonic() + try: + parts = urlsplit(repo_url) + if parts.scheme != "https" or not parts.hostname or parts.username or parts.password or parts.query or parts.fragment: + raise ValueError("credential-free HTTPS repository URL required") + if not re.fullmatch("[0-9a-f]{40}", commit): + raise ValueError("full lowercase commit SHA required") + archive = io.BytesIO() + with tarfile.open(fileobj=archive, mode="w"): + pass + # The empty checkout is filled by the first command inside the key-free task. + with SearchExecutorTask(lane, archive.getvalue(), deadline=deadline) as task: + record["task"] = task.name + + def command(step, name, args): + remaining = deadline - (time.monotonic() - started) - 2 + if remaining <= 0: + raise TimeoutError("smoke deadline") + timeout = min(300, remaining) + if name == "run": + args = {**args, "timeout_seconds": timeout} + before = time.monotonic() + result = task.client.submit( + name, args, timeout_seconds=timeout, limits={"memory_bytes": 2 * 1024**3, "disk_bytes": 2 * 1024**3} + ) + output_bytes = sum(result.get(k + "_bytes", len(result.get(k, "").encode())) for k in ("stdout", "stderr")) + if name != "run": + output_bytes = len(json.dumps(result).encode()) + row = { + "step": step, + "command": name, + "wall_seconds": result.get("wall_seconds", time.monotonic() - before), + "roundtrip_seconds": time.monotonic() - before, + "exit_code": result.get("exit_code"), + "output_bytes": output_bytes, + "scratch_peak_bytes": result.get("scratch_peak_bytes"), + "truncated": result.get("truncated", False), + } + if "input_bytes" in result: + row["input_bytes"] = result["input_bytes"] + record["commands"].append(row) + record["scratch_peak_bytes"] = max(record["scratch_peak_bytes"], result.get("scratch_peak_bytes", 0)) + if result.get("error") or result.get("timed_out") or result.get("status") == "could_not_build": + raise ValueError(step + " executor failure") + if name == "run" and result.get("exit_code") != 0: + raise ValueError(step + " nonzero or missing exit code") + if step == "install" and output_bytes == 0 and row["wall_seconds"] < 1: + raise ValueError("empty fast install: exit 0 with empty output in under 1 s") + return result + + command("clone", "run", {"command": ["sh", "-ec", CLONE, "clone", repo_url, commit]}) + files = command("list_files", "list_files", {}).get("files", []) + if not files: + raise ValueError("clone yielded zero files") + record["files_listed"] = len(files) + readme = next((p for p in files if Path(p).name.lower().startswith("readme")), None) + # Always open input even when the repository has no README. + read = command("read_file", "read_file", {"path": readme or files[0]}) + record["readme_present"] = readme is not None + record["input_bytes"] = read.get("input_bytes") + if not isinstance(record["input_bytes"], int): + raise ValueError("file read lacks input size") + command("grep", "grep", {"pattern": "test"}) + recipe, argv = install_recipe(files) + record["recipe"] = recipe + command("install", "run", {"command": argv}) + trivial = ["node", "-e", "console.log(1)"] if recipe == "npm" else ["python3", "-c", "print(1)"] + ran = command("trivial", "run", {"command": trivial}) + if ran.get("stdout", "").strip() != "1": + raise ValueError("trivial command did not print 1") + if not command("stop", "stop", {}).get("stopped"): + raise ValueError("stop not acknowledged") + record["status"] = "ok" + except Exception as exc: + record["status"] = "instrument_failure" + # Only our fixed diagnostic strings are safe; transport errors may contain URLs. + record["reason"] = ( + str(exc) + if isinstance(exc, ValueError) + and str(exc) + in { + "credential-free HTTPS repository URL required", + "full lowercase commit SHA required", + "no recognised install manifest", + "clone yielded zero files", + "file read lacks input size", + "trivial command did not print 1", + "stop not acknowledged", + "empty fast install: exit 0 with empty output in under 1 s", + *( + step + suffix + for step in ("clone", "list_files", "read_file", "grep", "install", "trivial", "stop") + for suffix in (" executor failure", " nonzero or missing exit code") + ), + } + else type(exc).__name__ + ) + record["wall_seconds"] = time.monotonic() - started + return record + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--lane", choices=("ax", "docker"), required=True) + parser.add_argument("--image", required=True) + parser.add_argument("--repo-url", required=True) + parser.add_argument("--commit", required=True) + parser.add_argument("--out", type=Path, required=True, help="One JSON record") + parser.add_argument("--deadline", type=float, default=900) + parser.add_argument("--ax-bin", default="ax") + parser.add_argument("--atespace", default="default") + parser.add_argument("--kubeconfig") + args = parser.parse_args(argv) + try: + validate_image(args.image) + lane = (DockerLane if args.lane == "docker" else AXLane)(args, Workload(args.image, frozenset(), args.deadline, None)) + record = run_smoke(lane, args.repo_url, args.commit, deadline=args.deadline) + except Exception as exc: + record = {"status": "instrument_failure", "reason": "lane setup: " + type(exc).__name__} + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n") + print(json.dumps(record)) + return int(record["status"] != "ok") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/pilot_select.py b/benchmarks/search/pilot_select.py new file mode 100644 index 0000000..60c65f6 --- /dev/null +++ b/benchmarks/search/pilot_select.py @@ -0,0 +1,360 @@ +"""Offline development-pair feasibility pilot; identities stay in a gitignored manifest. + +Run from the repository root with python -m benchmarks.search.pilot_select. +The existing reserved guard is executed as a black box: this module never reads v3. +""" + +from __future__ import annotations + +import argparse +import configparser +import hashlib +import json +import os +import re +import subprocess +from collections import Counter +from pathlib import Path +from urllib.parse import urlsplit + +from benchmarks.unseen import eligibility, ledger +from openultrasast.model.taxonomy import load_families + +LANGUAGES = ("python", "javascript", "typescript", "php", "java") +FAMILIES = ("injection", "path", "output_encoding", "untrusted_destination") +SHA = re.compile(r"[0-9a-f]{40}") +MANIFESTS = { + "requirements.txt": "pip", + "pyproject.toml": "pip", + "setup.py": "pip", + "Pipfile": "pip", + "package.json": "npm", + "composer.json": "composer", + "pom.xml": "maven", + "build.gradle": "gradle", + "build.gradle.kts": "gradle", +} + + +def encoded(value) -> bytes: + return (json.dumps(value, sort_keys=True, indent=2) + "\n").encode() + + +def safe(path: Path) -> bool: + return not eligibility.excluded(path) and not eligibility.excluded(path.resolve()) and not path.is_symlink() + + +def load_catalogs(root: Path) -> tuple[list[dict], dict]: + """Read raw metadata only; PairCase materialization would unnecessarily open source excerpts.""" + rows = [] + counts = {"files": 0, "bytes": 0, "rows": 0} + directory = (root / "benchmarks/pairs").resolve() + for path in sorted(directory.rglob("catalog.toml")): + if not safe(path) or not path.resolve().is_relative_to(directory): + continue + raw = path.read_bytes() + items = eligibility.parse(path, raw).get("pair", []) + if not isinstance(items, list) or not all(isinstance(row, dict) for row in items): + raise ValueError("invalid_catalog") + rows.extend(items) + counts["files"] += 1 + counts["bytes"] += len(raw) + counts["rows"] += len(items) + if not counts["bytes"] or not rows: + raise ValueError("empty_catalogs") + return rows, counts + + +def exclusions(root: Path, extra: tuple[Path, ...] = (), ledgers: tuple[Path, ...] = ()) -> tuple[set[str], dict]: + """Unseen-local reservations/used sets, including private journals; never the development used set. + + eligibility.build_used collects development data itself, so its union cannot exclude a + development pilot. The unseen usage ledger has opaque slices, not identities: every pool + is excluded in full regardless of usage, and ledger integrity is checked separately. + """ + directory = root / "benchmarks/unseen" + paths = { + p + for p in directory.rglob("*") + if p.suffix in {".toml", ".json", ".jsonl", ".yaml", ".yml"} and "fixtures" not in p.relative_to(directory).parts + } + paths.update(extra) + ledger_paths = set(ledgers) + ledger_paths.update(p for p in paths if any(word in p.stem.lower() for word in ("ledger", "usage"))) + paths.update(ledger_paths) + names: set[str] = set() + report = {"files": 0, "bytes": 0, "ledger_rows": 0} + for path in sorted(paths): + if not safe(path): + raise ValueError("unsafe_exclusion_source") + raw = path.read_bytes() + report["files"] += 1 + report["bytes"] += len(raw) + if path in ledger_paths: + report["ledger_rows"] += len(ledger.read(path)) + else: + value = eligibility.parse(path, raw) + + # TOML [[repository]] is a collection, whereas repositories() treats + # a scalar "repository" field as an identity. Walk containers too. + def collect(item): + names.update(eligibility.repositories(item)) + if isinstance(item, str) and eligibility.NAME.fullmatch(item): + names.add(eligibility.normalize(item)) + if isinstance(item, dict): + for child in item.values(): + if isinstance(child, (dict, list)): + collect(child) + elif isinstance(item, list): + for child in item: + collect(child) + + collect(value) + report.update(count=len(names), sha256=hashlib.sha256("\n".join(sorted(names)).encode()).hexdigest()) + return names, report + + +def cache_safe(path: Path) -> bool: + # Git config is intentionally readable here; every other exclusion and all + # symlink ancestors remain forbidden before opening files or listing trees. + lexical = path.absolute() + resolved = path.resolve() + return not any("v3" in part.lower() or part in {"__pycache__", ".venv"} for p in (lexical, resolved) for part in p.parts) and not any( + p.is_symlink() for p in (lexical, *lexical.parents) + ) + + +def enrich(rows: list[dict], caches: tuple[Path, ...], *, excluded: set[str] | None = None) -> dict: + """Prefer metadata actually present in a matching local clone, without fetching. + + This is only a preference: a cached checkout is not proof the pinned revision builds. + Only development-candidate paths are opened; unrelated cached repositories are not read. + """ + report = {"matched_checkouts": 0, "config_bytes": 0, "manifest_bytes": 0} + seen = {} + for row in rows: + try: + name = eligibility.normalize(row.get("repo", "")) + except (ValueError, TypeError): + continue + if name in (excluded or set()): + continue + if name not in seen: + seen[name] = [] + for cache in caches: + for checkout in (cache / name, cache / name.rsplit("/", 1)[-1]): + config = checkout / ".git/config" + if not cache_safe(config) or not config.is_file(): + continue + parser = configparser.ConfigParser() + raw = config.read_bytes() + parser.read_string(raw.decode()) + origin = parser.get('remote "origin"', "url", fallback="") + try: + matching = eligibility.normalize(origin) == name + except ValueError: + matching = False + if not matching: + continue + files = [] + for manifest in MANIFESTS: + path = checkout / manifest + if cache_safe(path) and path.is_file(): + report["manifest_bytes"] += len(path.read_bytes()) + files.append(manifest) + for folder in ("tests", "test", "spec", "__tests__", "src/test"): + directory = checkout / folder + if cache_safe(directory) and directory.is_dir() and any(directory.iterdir()): + files.append(folder + "/") + seen[name] = files + report["matched_checkouts"] += 1 + report["config_bytes"] += len(raw) + break + if seen[name]: + break + row["files"] = sorted(set(row.get("files", []) + seen[name])) + return report + + +def candidates(rows: list[dict], excluded: set[str]) -> tuple[list[dict], dict]: + taxonomy = load_families() + used = eligibility.UsedSet(sources={"unseen": excluded}) + out, rejected = [], Counter() + for row in rows: + reason = None + try: + raw_repo = row.get("repo", "") + repo = eligibility.normalize(raw_repo) + if "://" in raw_repo and not raw_repo.startswith("https://"): + raise ValueError + url = raw_repo if raw_repo.startswith("https://") else "https://github.com/" + repo + if repo.count("/") != 1: + raise ValueError + parsed = urlsplit(url) + if ( + parsed.netloc != "github.com" + or parsed.query + or parsed.fragment + or parsed.path.strip("/").removesuffix(".git").lower() != repo + ): + raise ValueError + if row.get("fix_repo") and eligibility.normalize(row["fix_repo"]) != repo: + raise ValueError + except (ValueError, TypeError): + rejected["repository"] += 1 + continue + if row.get("split", "train") != "train" or row.get("provenance") == "synthetic": + reason = "not_development_fix" + elif row.get("language") not in LANGUAGES: + reason = "language" + elif eligibility.rejection(eligibility.Resolution(repo, frozenset({repo}), repo), used, set()): + reason = "unseen" + vulnerable, fixed = row.get("parent", ""), row.get("commit", "") + if not all(isinstance(s, str) and SHA.fullmatch(s) for s in (vulnerable, fixed)) or vulnerable == fixed: + reason = reason or "commits" + expected = row.get("expected", []) + families = {row["family"]} if row.get("family") else set() + for item in [row, *expected]: + family = taxonomy.family_of_cwe(item.get("cwe", "")) + if family: + families.add(family.id) + families &= set(FAMILIES) + function = row.get("function") or next((r.get("function") for r in expected if r.get("function")), "") + file = row.get("path") or row.get("relpath", "") + if len(families) != 1: + reason = reason or "family" + if not function or function in {"", "", "*"} or not file: + reason = reason or "location" + if reason: + rejected[reason] += 1 + continue + files = row.get("files", []) + manifest = next((MANIFESTS[Path(f).name] for f in sorted(files) if Path(f).name in MANIFESTS), None) + tests = True if any(re.search(r"(^|/)(tests?|__tests__|spec)(/|$)|(^|/)test_[^/]+", f) for f in files) else None + out.append( + { + "repo": url.removesuffix(".git").rstrip("/"), + "vulnerable": vulnerable, + "fixed": fixed, + "family": next(iter(families)), + "language": row["language"], + "file": file, + "function": function, + "manifest": manifest, + "has_tests": tests, + } + ) + return out, dict(sorted(rejected.items())) + + +def select(rows: list[dict], *, seed: int, count: int = 20) -> list[dict]: + """Balance language/family representation, then prefer evidenced runnable projects. + + Seeded SHA ordering is invariant to catalog ordering and Python hash randomization. + At most one pair per normalized repository is selected. + """ + if count < 1: + raise ValueError("invalid_count") + remaining = sorted(rows, key=lambda row: hashlib.sha256(str(seed).encode() + encoded(row)).digest()) + selected, used = [], set() + languages, families = Counter(), Counter() + while remaining and len(selected) < count: + row = min( + remaining, + key=lambda r: ( + languages[r["language"]] >= 3, + languages[r["language"]], + families[r["family"]], + not (r["has_tests"] and r["manifest"]), + not r["manifest"], + not r["has_tests"], + ), + ) + selected.append(row) + used.add(eligibility.normalize(row["repo"])) + languages[row["language"]] += 1 + families[row["family"]] += 1 + remaining = [r for r in remaining if eligibility.normalize(r["repo"]) not in used] + return selected + + +def summary(rows: list[dict], *, seed: int, manifest: bytes, target: int = 20) -> dict: + languages = Counter(row["language"] for row in rows) + return { + "status": "selected" if len(rows) == target else "insufficient_candidates", + "purpose": "feasibility_only", + "seed": seed, + "target": target, + "selected": len(rows), + "gap_to_target": max(0, target - len(rows)), + "per_language": {lang: languages[lang] for lang in LANGUAGES}, + "per_family": {family: sum(r["family"] == family for r in rows) for family in FAMILIES}, + "language_floor_gaps": {lang: max(0, 3 - languages[lang]) for lang in LANGUAGES}, + "unknown_test_suite": sum(r["has_tests"] is None for r in rows), + "unknown_manifest": sum(r["manifest"] is None for r in rows), + "manifest_sha256": hashlib.sha256(manifest).hexdigest(), + "biases": ["known_vulnerable_location", "labelled_family", "runnable_project_preference"], + "qualification": "not_an_unseen_evaluation_or_block_admission", + } + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--seed", type=int, default=20261005) + parser.add_argument("--count", type=int, default=20) + parser.add_argument("--exclude", type=Path, action="append", default=[]) + parser.add_argument("--ledger", type=Path, action="append", default=[]) + parser.add_argument("--cache", type=Path, action="append", default=[]) + parser.add_argument("--manifest", type=Path, default=Path("benchmarks/search/private/pilot.json")) + parser.add_argument("--output", type=Path, default=Path("benchmarks/measurements/2026-10-05-search-pilot-selection/record.json")) + args = parser.parse_args(argv) + private = args.root / args.manifest + created = False + try: + if not private.resolve().is_relative_to((args.root / "benchmarks/search/private").resolve()): + raise ValueError("manifest_must_be_private") + ignored = subprocess.run(["git", "check-ignore", "-q", str(private)], cwd=args.root, capture_output=True) + if ignored.returncode != 0: + raise ValueError("manifest_not_gitignored") + guard = eligibility.verify_tests(args.root) + rows, instrument = load_catalogs(args.root) + excluded, exclusion_report = exclusions(args.root, tuple(args.exclude), tuple(args.ledger)) + # The black-box guard has already approved the development catalogs. + metadata = enrich(rows, tuple(args.cache) or (Path.home() / ".cache/openultrasast/repos",), excluded=excluded) + available, rejected = candidates(rows, excluded) + selected = select(available, seed=args.seed, count=args.count) + payload = encoded({"seed": args.seed, "pairs": selected}) + private.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with private.open("xb") as handle: + created = True + os.chmod(private, 0o600) + handle.write(payload) + # The URL-bearing private selection now lies in the guard's development tree, + # so even catalog bare-name aliases are checked without reading v3 ourselves. + guard = eligibility.verify_tests(args.root) + record = summary(selected, seed=args.seed, manifest=payload, target=args.count) + record.update( + catalogs=instrument, + exclusions=exclusion_report, + rejected=rejected, + local_metadata=metadata, + eligible_pairs=len(available), + eligible_repositories=len({r["repo"] for r in available}), + guard=guard, + ) + output = args.root / args.output + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("xb") as handle: + handle.write(encoded(record)) + print(encoded(record).decode(), end="") + return 0 if len(selected) == args.count else 2 + except Exception as exc: + if created: + private.unlink(missing_ok=True) + print(json.dumps({"status": "instrument_failure", "reason": type(exc).__name__})) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/ax/README.md b/ops/ax/README.md index 31b78e6..e1be9c3 100644 --- a/ops/ax/README.md +++ b/ops/ax/README.md @@ -228,3 +228,35 @@ pointing at your own bucket, a registry the workers can pull from, the runner im reachable as a Service, and the provider host allowed per task (which each task's EgressPolicy already does). The Model and Workspace manifests under `plane/` move unchanged; the Task templates get a new image reference; the kube context and `doctor`'s registry check are not configurable yet (the table above). Step by step: `docs/deployment.md`. + +## Search executor and verifier task contract (operator verified 2026-10-05) + +There are two workers and no queue. `no free workers available` is capacity +back-pressure: the dispatcher retries the same apply with 15–30 seconds of jitter, +up to the workload deadline. It does not retry a sandbox execution or route to the VM +because of this response. A shared reservation limits all AX workloads in one host +process to two tasks; separate host processes coordinate through AX's rejection. + +Search executors build on the disk-backed `/workspace`, export a single archive, and +are deleted before verifier repetitions start. The archive contains `checkout/`, +`products/`, and `spec.json` with a `none` build recipe. A verifier receives only +`VERIFY_INPUT_URL` and `RESULT_URL`; it never installs dependencies or fetches source. +The executor's host-scoped `PREPARED_PUT_URL` carries the built output. The host checks +its SHA-256 before passing it to fresh verifier tasks. Each side has three fresh runs. + +Verifier egress to `files.because-security.com:443` is requested but **pending**. +Until the operator enables it, live verifier input/output is blocked. The first input +download tolerates up to 60 seconds of delayed policy activation. + +`/` and `/tmp` consume the worker's 3 GiB RAM; `/workspace` has about 2 GiB usable disk. +Entrypoints set `TMPDIR=/workspace/tmp`, `npm_config_cache=/workspace/.npm`, +`MAVEN_OPTS=-Dmaven.repo.local=/workspace/.m2`, `COMPOSER_HOME=/workspace/.composer`, +`PIP_CACHE_DIR=/workspace/.pip`, and `GRADLE_USER_HOME=/workspace/.gradle`. +`OUSAST_SCRATCH_BYTES` overrides the default 2147483648-byte workspace guard. The guard +includes caches and open output logs, checks running children and completed writes, +and reports `could_not_build` / `scratch limit`. It is a monitored guard, not a hard +filesystem quota; programs that hardcode `/tmp` still consume the worker's RAM. + +Repository clones must use HTTPS `git clone https://github.com/...git` or codeload +archives, never `api.github.com`: tasks share one public IP. Search task inputs are +presigned checkout archives, so these entrypoints do not clone repositories themselves. diff --git a/src/openultrasast/search/executor.py b/src/openultrasast/search/executor.py index 071b34c..bd26624 100644 --- a/src/openultrasast/search/executor.py +++ b/src/openultrasast/search/executor.py @@ -124,7 +124,8 @@ class InProcessExecutor: raise ValueError("executor stopped") if command.seq != self.seq + 1: raise ValueError("out of order sequence") - self.deadline = time.monotonic() + command.timeout_seconds + started = time.monotonic() + self.deadline = started + command.timeout_seconds try: result = self._execute(command) except task_storage.ScratchLimit: @@ -132,6 +133,7 @@ class InProcessExecutor: result = {"status": "could_not_build", "reason": "scratch limit", "stopped": True} except (ValueError, OSError, KeyError, TypeError, TimeoutError) as exc: result = {"error": type(exc).__name__, "timed_out": isinstance(exc, TimeoutError)} + result["wall_seconds"] = time.monotonic() - started result = _bounded({"seq": command.seq, **result}, min(MAX_RESULT_BYTES, command.limits.get("result_bytes", MAX_RESULT_BYTES))) self.seq, self.last_digest, self.last_result = command.seq, digest, copy.deepcopy(result) return result @@ -256,7 +258,7 @@ class InProcessExecutor: timeout = min(timeout, self._remaining()) initial_bytes = self._tree_bytes() guard_root = task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else self.repo - task_storage.check(guard_root) + peak_bytes = task_storage.check(guard_root) with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err: proc = subprocess.Popen( argv, cwd=self.repo, env=clean_environment(), stdout=out, stderr=err, start_new_session=True, preexec_fn=constrain @@ -268,13 +270,21 @@ class InProcessExecutor: # guard is not a replacement for the operator's memory limit. try: while True: - task_storage.check(guard_root, additional_bytes=os.fstat(out.fileno()).st_size + os.fstat(err.fileno()).st_size) + peak_bytes = max( + peak_bytes, + task_storage.check(guard_root, additional_bytes=os.fstat(out.fileno()).st_size + os.fstat(err.fileno()).st_size), + ) if self._tree_bytes() - initial_bytes > limits.get("disk_bytes", task_storage.DEFAULT_SCRATCH_BYTES): disk_exceeded = True break try: proc.wait(timeout=min(0.05, self._remaining())) - task_storage.check(guard_root, additional_bytes=os.fstat(out.fileno()).st_size + os.fstat(err.fileno()).st_size) + peak_bytes = max( + peak_bytes, + task_storage.check( + guard_root, additional_bytes=os.fstat(out.fileno()).st_size + os.fstat(err.fileno()).st_size + ), + ) if self._tree_bytes() - initial_bytes > limits.get("disk_bytes", task_storage.DEFAULT_SCRATCH_BYTES): disk_exceeded = True break @@ -295,6 +305,9 @@ class InProcessExecutor: "timed_out": timed_out, "disk_limit_exceeded": disk_exceeded, "aggregate_disk_isolation": "outer-task-quota-required", + "scratch_peak_bytes": peak_bytes, + "stdout_bytes": os.fstat(out.fileno()).st_size, + "stderr_bytes": os.fstat(err.fileno()).st_size, "stdout": out.read(16384).decode(errors="replace"), "stderr": err.read(16384).decode(errors="replace"), "truncated": os.fstat(out.fileno()).st_size > 16384 or os.fstat(err.fileno()).st_size > 16384, diff --git a/tests/test_search_executor_smoke.py b/tests/test_search_executor_smoke.py new file mode 100644 index 0000000..c0379e0 --- /dev/null +++ b/tests/test_search_executor_smoke.py @@ -0,0 +1,162 @@ +"""No-network end-to-end executor smoke, through the real mailbox client.""" + +import json +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from benchmarks.ax import batch +from benchmarks.search import executor_smoke as smoke +from openultrasast.search.executor import Command, InProcessExecutor + + +class Store: + def __init__(self, executor): + self.executor, self.objects = executor, {} + + def _put(self, key, data): + self.objects[key] = data + + def _delete(self, key): + self.objects.pop(key, None) + + def presign_get(self, key, expires): + return "https://files.example/" + key + + presign_put = presign_get + + def put(self, url, data, timeout): + self.result = json.dumps(self.executor.execute(Command(**json.loads(data)))).encode() + + def get(self, url, timeout): + return self.result + + +class Executor(InProcessExecutor): + empty = False + empty_install = False + + def _task_run(self, argv, timeout, limits): + if argv[:2] == ["sh", "-ec"]: + if not self.empty: + (self.repo / "README.md").write_text("test input") + (self.repo / "requirements.txt").write_text("") + return dict(exit_code=0, stdout="cloned", stderr="", scratch_peak_bytes=4096, stdout_bytes=6, stderr_bytes=0) + if "pip" in argv: + return dict( + exit_code=0, + stdout="" if self.empty_install else "installed", + stderr="", + scratch_peak_bytes=8192, + stdout_bytes=0 if self.empty_install else 9, + stderr_bytes=0, + ) + return super()._task_run(argv, timeout, limits) + + +def setup(tmp_path, monkeypatch): + executor = Executor(tmp_path, task_boundary=True) + store = Store(executor) + monkeypatch.setattr("openultrasast.search.executor.URLTransport", lambda: store) + events = [] + lane = SimpleNamespace( + store=store, + host="files.example", + args=SimpleNamespace(atespace="default"), + workload=batch.Workload("image@sha256:" + "a" * 64, frozenset(), 900, None), + clock=time.monotonic, + ) + + def ax(*args, **kwargs): + events.append(args[0]) + if args[0] == "apply": + doc = kwargs["manifest"] + assert doc["metadata"]["name"].startswith("ousast-engine-search-exec-") + assert {x["name"] for x in doc["spec"]["env"]} == {"REPO_URL", "COMMAND_GET_URL", "EXECUTOR_RESULT_PUT_URL", "PREPARED_PUT_URL"} + + lane.ax = ax + return executor, lane, events + + +def test_smoke_mailbox_end_to_end(tmp_path, monkeypatch): + executor, lane, events = setup(tmp_path, monkeypatch) + record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40) + assert record["status"] == "ok" + assert executor.stopped + assert events == ["apply", "delete"] + assert not lane.store.objects + assert [r["step"] for r in record["commands"]] == ["clone", "list_files", "read_file", "grep", "install", "trivial", "stop"] + assert record["scratch_peak_bytes"] >= 8192 + read = next(r for r in record["commands"] if r["step"] == "read_file") + assert read["input_bytes"] == 10 + assert all(r["wall_seconds"] >= 0 and r["output_bytes"] >= 0 for r in record["commands"]) + + +@pytest.mark.parametrize("kind,reason", [("empty", "clone yielded zero files"), ("empty_install", "empty fast install")]) +def test_instrument_failure_and_cleanup(tmp_path, monkeypatch, kind, reason): + executor, lane, events = setup(tmp_path, monkeypatch) + setattr(executor, kind, True) + record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40) + assert record["status"] == "instrument_failure" + assert reason in record["reason"] + assert events[-1] == "delete" + assert not lane.store.objects + + +@pytest.mark.parametrize( + "files,recipe", [(["package.json"], "npm"), (["composer.json"], "composer"), (["pom.xml"], "maven"), (["pyproject.toml"], "pip")] +) +def test_recipes(files, recipe): + assert smoke.install_recipe(files)[0] == recipe + + +def test_real_executor_measurements(tmp_path): + (tmp_path / "input").write_bytes(b"12345") + result = InProcessExecutor(tmp_path, task_boundary=True).submit("run", {"command": ["python3", "-c", "print(1)"]}) + assert result["exit_code"] == 0 + assert result["stdout_bytes"] == 2 + assert result["scratch_peak_bytes"] >= 5 + assert result["wall_seconds"] > 0 + + +def test_docker_managed_task_uses_docker_and_no_heap_flag(tmp_path, monkeypatch): + import subprocess + + executor = Executor(tmp_path, task_boundary=True) + store = Store(executor) + monkeypatch.setattr("openultrasast.search.executor.URLTransport", lambda: store) + monkeypatch.setenv("S3_ENDPOINT", "https://files.example") + calls = [] + + def runner(argv, **kwargs): + calls.append(argv) + return subprocess.CompletedProcess(argv, 0, "", "") + + lane = batch.DockerLane( + SimpleNamespace(atespace="default"), batch.Workload("image@sha256:" + "a" * 64, frozenset(), 900, None), store=store, runner=runner + ) + record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40) + assert record["status"] == "ok" + assert [a[1] for a in calls] == ["run", "rm"] + assert "--heap-profile" not in calls[0] + assert calls[1][2] == "-f" + + +def test_apply_exception_still_deletes(tmp_path, monkeypatch): + executor, lane, events = setup(tmp_path, monkeypatch) + + def failing(*args, **kwargs): + events.append(args[0]) + if args[0] == "apply": + raise OSError("do not expose https://secret.example/token") + + lane.ax = failing + result = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40) + assert result["status"] == "instrument_failure" + assert events == ["apply", "delete"] + assert "secret.example" not in json.dumps(result) diff --git a/tests/test_search_pilot_select.py b/tests/test_search_pilot_select.py new file mode 100644 index 0000000..300062f --- /dev/null +++ b/tests/test_search_pilot_select.py @@ -0,0 +1,175 @@ +"""Offline pilot selection: synthetic repositories only.""" + +import json +from pathlib import Path + +import pytest + +with pytest.MonkeyPatch.context() as patch: + patch.syspath_prepend(str(Path(__file__).resolve().parents[1])) + from benchmarks.search import pilot_select as pilot + + +def pair(i, language="python", family="injection", **extra): + return { + "repo": f"https://github.com/synthetic/project-{i}", + "parent": "a" * 40, + "commit": "b" * 40, + "language": language, + "family": family, + "split": "train", + "path": "src/main.py", + "expected": [{"function": "handler"}], + **extra, + } + + +def test_balanced_unique_deterministic_selection(): + rows = [pair(f"{lang}-{i}", lang, pilot.FAMILIES[i % 4]) for lang in pilot.LANGUAGES for i in range(8)] + candidates, rejected = pilot.candidates(rows, set()) + selected = pilot.select(candidates, seed=17) + assert len(selected) == len({r["repo"] for r in selected}) == 20 + assert all(sum(r["language"] == lang for r in selected) >= 3 for lang in pilot.LANGUAGES) + assert set(r["family"] for r in selected) == set(pilot.FAMILIES) + assert selected == pilot.select(list(reversed(candidates)), seed=17) + assert selected != pilot.select(candidates, seed=18) + assert not rejected + + +def test_filter_non_development_bad_commits_families_and_used(): + rows = [ + pair(1), + pair(2, split="holdout"), + pair(3, family="access_control"), + pair(4, parent="b" * 40), + pair(5), + pair(6, parent="bad"), + pair(7, repo="http://github.com/synthetic/no"), + ] + selected, rejected = pilot.candidates(rows, {"synthetic/project-5"}) + assert len(selected) == 1 + assert sum(rejected.values()) == 6 + + +def test_prefer_evidenced_test_suite_and_manifest_and_unique_repos(): + rows = [pair(1), pair(2, files=["pyproject.toml", "tests/test_main.py"]), pair(2, family="path")] + selected = pilot.select(pilot.candidates(rows, set())[0], seed=2, count=2) + assert selected[0]["repo"].endswith("project-2") + assert len({row["repo"] for row in selected}) == 2 + assert selected[0]["has_tests"] and selected[0]["manifest"] == "pip" + + +def test_unseen_local_pool_used_and_ledger(tmp_path): + unseen = tmp_path / "benchmarks/unseen" + unseen.mkdir(parents=True) + (unseen / "pool-p1.toml").write_text('[[repository]]\nrepo="https://github.com/synthetic/project-1"\n') + (unseen / "used.json").write_text(json.dumps({"repo": "synthetic/project-2"})) + names, report = pilot.exclusions(tmp_path) + assert names == {"synthetic/project-1", "synthetic/project-2"} + assert report["files"] == 2 and report["bytes"] > 0 + (unseen / "usage.jsonl").write_text('{"bad":true}\n') + with pytest.raises(ValueError, match="broken_digest_chain"): + pilot.exclusions(tmp_path) + + +def test_never_open_reserved_or_symlink_catalogs(tmp_path): + root = tmp_path / "benchmarks/pairs" + root.mkdir(parents=True) + (root / "catalog.toml").write_text('[[pair]]\nrepo="synthetic/one"\n') + hidden = root / "population-v3" + hidden.mkdir() + (hidden / "catalog.toml").write_bytes(b"not toml") + (root / "alias").symlink_to(hidden, target_is_directory=True) + rows, counts = pilot.load_catalogs(tmp_path) + assert len(rows) == 1 and counts["files"] == 1 + assert counts["bytes"] > 0 + + +def test_summary_contains_counts_and_digests_not_names(): + selected = pilot.select(pilot.candidates([pair(1)], set())[0], seed=1) + summary = pilot.summary(selected, seed=1, manifest=b"private") + assert summary["selected"] == 1 and summary["gap_to_target"] == 19 + assert "synthetic" not in json.dumps(summary) + assert summary["unknown_test_suite"] == 1 + + +def test_bare_used_names_and_repository_alias_fields(tmp_path): + unseen = tmp_path / "benchmarks/unseen" + unseen.mkdir(parents=True) + (unseen / "used.json").write_text(json.dumps({"used": ["synthetic/project-1"]})) + assert pilot.exclusions(tmp_path)[0] == {"synthetic/project-1"} + + +def test_cross_repository_and_noncanonical_urls_rejected(): + rows = [ + pair(1, fix_repo="synthetic/other"), + pair(2, repo="https://user:secret@github.com/synthetic/project-2"), + pair(3, repo="https://github.com/synthetic/project-3?query=1"), + pair(4, repo="https://github.com/synthetic/project-4/extra"), + ] + assert not pilot.candidates(rows, set())[0] + + +def test_metadata_from_matching_local_checkout_only(tmp_path): + checkout = tmp_path / "project-1" + (checkout / ".git").mkdir(parents=True) + (checkout / ".git/config").write_text('[remote "origin"]\nurl=https://github.com/synthetic/project-1\n') + (checkout / "pyproject.toml").write_text('[project]\nname="synthetic"\n') + (checkout / "tests").mkdir() + (checkout / "tests/test_main.py").write_text("def test_smoke(): assert True\n") + rows = [pair(1), pair(2)] + counts = pilot.enrich(rows, (tmp_path,)) + assert counts["matched_checkouts"] == 1 + selected = pilot.candidates(rows, set())[0] + assert selected[0]["manifest"] == "pip" and selected[0]["has_tests"] is True + assert selected[1]["manifest"] is None + + +def test_cache_enrichment_never_follows_reserved_paths(tmp_path, monkeypatch): + cache = tmp_path / "cache" + checkout = cache / "project-1" + (checkout / ".git").mkdir(parents=True) + reserved = tmp_path / "population-v3" + reserved.mkdir() + config = reserved / "config" + config.write_text('[remote "origin"]\nurl=https://github.com/synthetic/project-1\n') + (checkout / ".git/config").symlink_to(config) + original = Path.read_bytes + + def guarded(path): + if "population-v3" in path.resolve().parts: + pytest.fail("reserved file opened") + return original(path) + + monkeypatch.setattr(Path, "read_bytes", guarded) + assert pilot.enrich([pair(1)], (cache,))["matched_checkouts"] == 0 + (checkout / ".git/config").unlink() + (checkout / ".git/config").write_text(config.read_text()) + (checkout / "tests").symlink_to(reserved, target_is_directory=True) + (checkout / "pyproject.toml").symlink_to(config) + rows = [pair(1)] + pilot.enrich(rows, (cache,)) + assert not rows[0]["files"] + + +def test_enrichment_skips_unseen_before_opening_cache(tmp_path, monkeypatch): + def fail(*args): + pytest.fail("unseen cache read") + + monkeypatch.setattr(Path, "read_bytes", fail) + assert pilot.enrich([pair(1)], (tmp_path,), excluded={"synthetic/project-1"})["matched_checkouts"] == 0 + + +def test_cache_enrichment_refuses_ancestor_symlinks(tmp_path, monkeypatch): + reserved = tmp_path / "population-v3" + checkout = reserved / "project-1" + (checkout / ".git").mkdir(parents=True) + (checkout / ".git/config").write_text('[remote "origin"]\nurl=https://github.com/synthetic/project-1\n') + alias = tmp_path / "alias" + alias.symlink_to(reserved, target_is_directory=True) + + def fail(*args): + pytest.fail("reserved ancestor opened") + + monkeypatch.setattr(Path, "read_bytes", fail) + assert pilot.enrich([pair(1)], (alias,))["matched_checkouts"] == 0