merge(search-with-proof): build recipes and stderr-bearing failures [skip ci]

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/ax/batch.py

@@ -724,7 +724,11 @@ class SearchExecutorTask:
         remaining = min(300, self.end - time.monotonic())
         result = self.client.submit("prepare_verification", spec, timeout_seconds=remaining)
         if result.get("status") != "ok":
-            raise ValueError(result.get("reason", "could_not_build"))
+            from openultrasast.search.task_storage import CommandFailure
+
+            raise CommandFailure(
+                result.get("phase", "build"), result.get("exit_code"), result.get("stderr", ""), result.get("reason", "could_not_build")
+            )
         stored = bounded_read(lambda: self.lane.store._get(self.keys[3]), self.end - time.monotonic())
         if stored is None or hashlib.sha256(stored[0]).hexdigest() != result.get("sha256"):
             raise ValueError("executor archive missing or digest mismatch")

benchmarks/ax/search_verify.py

@@ -110,4 +110,7 @@ class AXSideDispatcher:
             max(r.scratch_peak_bytes for r in records),
             isolation_mode="task-boundary",
             chromium_no_sandbox=any(r.chromium_no_sandbox for r in records),
+            phase=last.phase,
+            exit_code=last.exit_code,
+            stderr=last.stderr,
         )

benchmarks/search/pilot_run.py

@@ -123,7 +123,7 @@ class Pilot:
         self.sides = None
         self.verifications = []
 
-    def failure(self, phase, reason):
+    def failure(self, phase, reason, command=None):
         private = tuple(str(v) for k, v in self.row.items() if k in {"repo", "file", "function", "vulnerable", "fixed"})
         private += (str(self.root), self.row["repo"].removeprefix("https://github.com/"))
         if isinstance(reason, BaseException):
@@ -135,7 +135,15 @@ class Pilot:
             safe = kind + ": " + diagnostic(message, maximum=max(0, 298 - len(kind)), private=private)
         else:
             safe = diagnostic(reason, private=private)
-        self.record["failures"].append(dict(phase=phase, reason=safe))
+        entry = dict(phase=phase, reason=safe)
+        command = command if command is not None else reason
+        if hasattr(command, "exit_code"):
+            entry.update(
+                command_phase=diagnostic(command.phase),
+                exit_code=command.exit_code,
+                stderr=diagnostic(command.stderr, maximum=1500, private=private),
+            )
+        self.record["failures"].append(entry)
 
     @contextmanager
     def executor(self, revision, deadline=900):
@@ -300,7 +308,7 @@ class Pilot:
 
                 for side in result.sides:
                     if side.outcome != "observed":
-                        self.failure("verify/" + side.outcome, "side_verification_failed")
+                        self.failure("verify/" + side.outcome, side.reason, command=side)
                 if result.outcome != "demonstrated":
                     self.failure("verify", "no_consistent_differential" if result.outcome == "inconclusive" else result.outcome)
                 end = result.outcome

src/openultrasast/search/demo.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import json
 import re
+import tomllib
 from pathlib import Path, PurePosixPath
 from typing import Any, cast
 
@@ -145,6 +146,75 @@ def checkout_path(value: Any) -> str:
     return value
 
 
+_MANIFESTS = {
+    "pip": ("pyproject.toml", "setup.py", "setup.cfg"),
+    "npm": ("package.json", "package-lock.json"),
+    "composer": ("composer.json", "composer.lock"),
+    "maven": ("pom.xml",),
+    "gradle": ("build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts", "gradlew"),
+}
+
+
+def build_project(recipe: str, path: Path, checkout: Path) -> Path:
+    """Validate the recipe ecosystem against actual checkout files."""
+
+    def manifest(candidate: Path) -> bool:
+        return candidate.name in _MANIFESTS[recipe] or (recipe == "pip" and candidate.match("requirements*.txt"))
+
+    def contained(candidate: Path) -> bool:
+        return candidate.resolve().is_relative_to(checkout.resolve())
+
+    if not contained(path):
+        raise ValueError("recipe input outside checkout")
+    if path.is_file():
+        if not manifest(path):
+            raise ValueError("recipe input is not an ecosystem manifest")
+        return path.parent
+    if path.is_dir() and any(p.is_file() and contained(p) and manifest(p) for p in path.iterdir()):
+        return path
+    raise ValueError("recipe input must be an ecosystem manifest or project directory")
+
+
+def preparation_command(recipe: str, path: Path, checkout: Path, products: Path) -> tuple[list[str], Path]:
+    project = build_project(recipe, path, checkout)
+    if recipe == "pip":
+        requirements = path if path.is_file() and path.match("requirements*.txt") else None
+        pyproject = project / "pyproject.toml"
+        backend = False
+        if requirements is None and pyproject.is_file():
+            if not pyproject.resolve().is_relative_to(checkout.resolve()):
+                raise ValueError("recipe input outside checkout")
+            backend = bool(tomllib.loads(pyproject.read_text()).get("build-system", {}).get("build-backend"))
+        if requirements is None and not backend and not (project / "setup.py").is_file():
+            fallback = project / "requirements.txt"
+            if fallback.is_file():
+                requirements = fallback
+        target = requirements or project
+        if not target.resolve().is_relative_to(checkout.resolve()):
+            raise ValueError("recipe input outside checkout")
+        return [
+            "python3",
+            "-I",
+            "-m",
+            "pip",
+            "install",
+            "--no-deps",
+            "--target",
+            str(products / "packages"),
+            *(["-r", str(target)] if requirements else [str(target)]),
+        ], project
+    if recipe == "npm":
+        return ["npm", "ci" if (project / "package-lock.json").is_file() else "install", "--ignore-scripts"], project
+    if recipe == "composer":
+        return ["composer", "install", "--no-interaction", "--no-plugins", "--no-scripts"], project
+    if recipe == "maven":
+        return ["mvn", "-q", "-DskipTests", "package", "-f", str(project / "pom.xml")], project
+    wrapper = project / "gradlew"
+    if not wrapper.is_file() or not wrapper.resolve().is_relative_to(checkout.resolve()):
+        raise ValueError("missing or external Gradle wrapper")
+    return ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(project), "assemble"], project
+
+
 def _text(value: Any, names: set[str]) -> str:
     if not isinstance(value, str) or "\0" in value or len(value.encode()) > MAX_VALUE_BYTES:
         raise ValueError("invalid or oversized value")
@@ -163,7 +233,7 @@ def _arguments(value: Any, names: set[str]) -> None:
         _text(arg, names)
 
 
-def validate_demo(value: Any) -> dict[str, Any]:
+def validate_demo(value: Any, *, checkout: Path | None = None) -> dict[str, Any]:
     try:
         encoded = json.dumps(value, allow_nan=False).encode()
     except (ValueError, TypeError, RecursionError) as exc:
@@ -186,8 +256,10 @@ def validate_demo(value: Any) -> dict[str, Any]:
     else:
         if len(args) != 1:
             raise ValueError("recipe requires one checkout path")
-        if not (recipe in ("npm", "composer", "gradle") and args[0] == "."):
+        if args[0] != ".":
             checkout_path(args[0])
+        if checkout is not None:
+            build_project(recipe, checkout / args[0], checkout)
     _keys(start, {"runtime", "path", "arguments", "mode"}, {"port"})
     if not isinstance(start["runtime"], str) or start["runtime"] not in RUNTIMES:
         raise ValueError("unknown runtime")

src/openultrasast/search/executor.py

@@ -269,12 +269,12 @@ class InProcessExecutor:
             "isolation_mode": self.isolation_mode,
             "exit_code": result.exit_code,
             "stdout": result.stdout[:16384],
-            "stderr": task_storage.diagnostic(result.stderr, maximum=4000),
+            "stderr": task_storage.diagnostic(result.stderr, maximum=1500),
             "timed_out": result.timed_out,
             "truncated": len(result.stdout) > 16384 or len(result.stderr) > 16384,
         }
 
-    def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]:
+    def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any], *, cwd: Path | None = None) -> dict[str, Any]:
         process_limit = None
         try:
             uid = os.getuid()
@@ -311,7 +311,7 @@ class InProcessExecutor:
         self.phase = "spawn"
         with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
             proc = subprocess.Popen(
-                argv, cwd=self.repo, env=clean_environment(), stdout=out, stderr=err, start_new_session=True, preexec_fn=constrain
+                argv, cwd=cwd or self.repo, env=clean_environment(), stdout=out, stderr=err, start_new_session=True, preexec_fn=constrain
             )
             self.phase = "run"
             timed_out = False
@@ -358,7 +358,7 @@ class InProcessExecutor:
                 proc.wait()
             out.seek(0)
             err.seek(max(0, os.fstat(err.fileno()).st_size - 16384))
-            stderr = task_storage.diagnostic(err.read().decode(errors="replace"), maximum=4000)
+            stderr = task_storage.diagnostic(err.read().decode(errors="replace"), maximum=1500)
             phase = "scratch guard" if disk_exceeded else "timeout" if timed_out else self.phase
             if disk_exceeded:
                 self.stopped = True
@@ -389,14 +389,14 @@ class InProcessExecutor:
         """Build and export through the host-scoped output URL, never a model URL."""
         import shutil
 
-        from .demo import BUILD_RECIPES, validate_demo
+        from .demo import preparation_command, validate_demo
         from .verify import _tree
         from .verify_task import pack_checkout, validate_spec
 
         if not self.task_boundary or not self.prepared_put_url:
             raise ValueError("preparation requires an executor task output")
         spec = copy.deepcopy(spec)
-        validate_demo(spec["demo"])
+        validate_demo(spec["demo"], checkout=self.repo)
         recipe = spec["demo"]["build"]
         spec["demo"]["build"] = {"recipe": "none", "arguments": []}
         validate_spec(spec)
@@ -406,22 +406,16 @@ class InProcessExecutor:
             products.mkdir()
             if recipe["recipe"] != "none":
                 path = self._path(recipe["arguments"][0])
-                argv = [
-                    arg.replace("/scratch", str(products))
-                    for arg in BUILD_RECIPES[recipe["recipe"]]
-                    if arg not in {"--offline", "--no-index"}
-                ]
-                argv.append(str(path))
-                if recipe["recipe"] == "maven":
-                    argv += ["package", "-DskipTests"]
-                elif recipe["recipe"] == "gradle":
-                    # The project pins its Gradle version through its wrapper.
-                    # Invoke with sh because checkout archives may lose execute bits.
-                    wrapper = self._path(str(path.relative_to(self.repo) / "gradlew"))
-                    argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"]
-                built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES, **limits})
+                argv, project = preparation_command(recipe["recipe"], path, self.repo, products)
+                self.phase = "build"
+                built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES, **limits}, cwd=project)
                 if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
-                    return {**built, "status": "could_not_build"}
+                    return {
+                        **built,
+                        "status": "could_not_build",
+                        "phase": "build",
+                        "stderr": task_storage.diagnostic(built.get("stderr", ""), maximum=1500),
+                    }
             # Validate internal generated links before materializing them as
             # regular archive members. No host/external files may be exported.
             _tree(self.repo, generated_links=True)

src/openultrasast/search/task_storage.py

@@ -106,8 +106,12 @@ def diagnostic(value: object, *, maximum: int = 300, private: tuple[str, ...] =
     text = re.sub(r"(?i)bearer\s+[^\s,;\"']+", "Bearer [redacted]", text)
     text = re.sub(r"(?i)(?:https?|s3)://[^\s<>\"']+", "[redacted-url]", text)
     text = re.sub(
-        r"(?i)([\w-]*(?:authorization|signature|credential|token|secret|password|key))\s*[:=]\s*[^\s,;&\"']+", r"\1=[redacted]", text
+        r"(?i)([\w-]*(?:authorization|signature|credential|token|secret|password|key))[\"']?\s*[:=]\s*[\"']?[^\s,;&\"']+",
+        r"\1=[redacted]",
+        text,
     )
+    text = re.sub(r"\b(?:gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+|sk-[A-Za-z0-9_-]+|npm_[A-Za-z0-9]+)\b", "[redacted]", text)
+    text = re.sub(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b", "[redacted]", text)
     for name in sorted(set(private), key=len, reverse=True):
         if name:
             text = text.replace(name, "[repository]")
@@ -125,3 +129,14 @@ def exception_reason(exc: BaseException, *, private: tuple[str, ...] = ()) -> st
         half = max(0, (maximum - 2) // 2)
         return prefix + diagnostic(original, maximum=half, private=private) + "; " + diagnostic(cleanup, maximum=half, private=private)
     return prefix + diagnostic(exc, maximum=maximum, private=private)
+
+
+class CommandFailure(ValueError):
+    """Transport a bounded command diagnostic without dropping stderr."""
+
+    def __init__(self, phase: str, exit_code: int | None, stderr: str, reason: str = "") -> None:
+        self.phase = phase
+        self.exit_code = exit_code
+        self.stderr = diagnostic(stderr, maximum=1500)
+        summary = diagnostic(reason) if reason else f"{phase}: exit code {exit_code}"
+        super().__init__(summary + (": " + self.stderr if self.stderr else ""))

src/openultrasast/search/verify.py

@@ -30,7 +30,7 @@ from . import _sandbox, task_storage
 from .budget import SearchBudget
 from .demo import BUILD_RECIPES, FAMILY_ORACLES, RUNTIMES, capture_output, load_demo, substitute, validate_oracle
 from .oracles import BrowserExecutor, Canary, Oracle, oracle_for
-from .task_storage import exception_reason
+from .task_storage import CommandFailure, diagnostic, exception_reason
 
 
 @dataclass(frozen=True)
@@ -62,6 +62,9 @@ class SideRecord:
     scratch_peak_bytes: int = 0
     isolation_mode: str = field(default_factory=_sandbox.isolation_mode)
     chromium_no_sandbox: bool = False
+    phase: str = ""
+    exit_code: int | None = None
+    stderr: str = ""
 
 
 @dataclass(frozen=True)
@@ -133,18 +136,19 @@ def _tree(root: Path, *, generated_links: bool = False) -> str:
 # Executed as verifier-owned code inside the same network sandbox as the app.
 # Redirects are refused: only the configured loopback application is reachable.
 _HTTP_DRIVER = r"""
-import http.client, importlib.util, json, os, socket, subprocess, sys, time
+import http.client, importlib.util, json, os, socket, subprocess, sys, tempfile, time
 spec = importlib.util.spec_from_file_location('demo', '/verifier-demo.py')
 demo = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(demo)
 config = json.load(sys.stdin)
 schema = config['schema']
-app = subprocess.Popen(config['command'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+app_errors = tempfile.TemporaryFile()
+app = subprocess.Popen(config['command'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=app_errors)
 try:
     until = time.monotonic() + config['timeout'] / 2
     while True:
         if app.poll() is not None:
-            raise RuntimeError('HTTP application exited during readiness')
+            raise SystemExit(app.returncode or 1)
         try:
             connection = socket.create_connection(('127.0.0.1', schema['start']['port']), timeout=0.1)
             connection.close()
@@ -183,6 +187,9 @@ finally:
     except subprocess.TimeoutExpired:
         app.kill()
         app.wait()
+    app_errors.seek(max(0, app_errors.seek(0, 2) - 16384))
+    sys.stderr.write(app_errors.read().decode(errors="replace"))
+    app_errors.close()
 """
 
 
@@ -223,7 +230,12 @@ def _http_steps(
             acknowledged = True
         result = future.result()
     if not acknowledged or result.exit_code or result.timed_out:
-        raise ValueError("HTTP application/steps failed or readiness not acknowledged: " + result.stderr[-2000:])
+        raise CommandFailure(
+            "start" if not acknowledged else "run",
+            result.exit_code,
+            result.stderr,
+            "HTTP application/steps failed or readiness not acknowledged",
+        )
     outputs = json.loads(result.stdout)
     if not isinstance(outputs, list) or len(outputs) != len(schema["steps"]) or not all(isinstance(value, str) for value in outputs):
         raise ValueError("invalid HTTP driver output")
@@ -249,18 +261,21 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
             if root is not None:
                 disk_peak = max(disk_peak, sum(p.lstat().st_blocks * 512 for p in root.rglob("*")))
 
-    def record(outcome: str, reason: str) -> SideRecord:
+    def record(outcome: str, reason: str, phase: str = "", result: Any = None) -> SideRecord:
         return SideRecord(
             outcome,
             tuple(observations),
             time.monotonic() - started,
-            reason,
+            diagnostic(reason, maximum=1500),
             timings["build"],
             timings["ready"],
             timings["run"],
             disk_peak,
             isolation_mode="task-boundary" if task_boundary else _sandbox.isolation_mode(),
             chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)),
+            phase=phase,
+            exit_code=result.exit_code if result is not None else None,
+            stderr=diagnostic(result.stderr, maximum=1500) if result is not None else "",
         )
 
     try:
@@ -305,7 +320,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                         command += ("assemble",)
                     compiled = timed("build", runner, job(command), scratch=products, mounts=mounts, writable_checkout=True)
                     if compiled.exit_code or compiled.timed_out:
-                        return record("could_not_build", "build recipe failed: " + compiled.stderr[-2000:])
+                        return record("could_not_build", "build recipe failed", "build", compiled)
                 if tuple(_tree(p) for p in immutable) != before:
                     return record("inconclusive", "build changed immutable inputs")
                 product_hash = _tree(products, generated_links=True)
@@ -354,7 +369,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                             namespace_pid=oracle.namespace_pid,
                         )
                         if ready.exit_code or ready.timed_out:
-                            return record("could_not_run", "app readiness failed: " + ready.stderr[-2000:])
+                            return record("could_not_run", "app readiness failed", "start", ready)
                     if family == "ssrf" and oracle.observe()[0]:
                         return record("inconclusive", "readiness triggered the SSRF oracle")
                     # Readiness cannot supply the attack's evidence or leave a command marker.
@@ -401,7 +416,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                             if result.timed_out:
                                 return record("inconclusive", "app attack timed out")
                             if result.exit_code:
-                                return record("could_not_run", "app invocation failed")
+                                return record("could_not_run", "app invocation failed", "run", result)
                             timed("run", oracle.capture, result.stdout)
                             capture_output(step, result.stdout, captures, canary.nonce)
                     observed, evidence = timed("run", oracle.observe)
@@ -416,6 +431,8 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                 finally:
                     oracle.close()
         return record("observed", "fresh task run completed" if task_boundary else "three fresh runs completed")
+    except CommandFailure as exc:
+        return record("could_not_run", str(exc), exc.phase, exc)
     except task_storage.ScratchLimit as exc:
         return record("could_not_build", exception_reason(exc))
     except (OSError, ValueError, AssertionError, KeyError, IndexError, TypeError) as exc:
@@ -472,7 +489,7 @@ def _run_task(
                 os.killpg(proc.pid, signal.SIGKILL)
             proc.wait()
         out.seek(0)
-        err.seek(0)
+        err.seek(max(0, os.fstat(err.fileno()).st_size - 16384))
         return SandboxResult(
             proc.returncode, out.read(1048576).decode(errors="replace"), err.read(1048576).decode(errors="replace"), expired
         )

src/openultrasast/search/worker.py

@@ -81,7 +81,12 @@ DEMO_GUIDE = (
     'Demo schema: {"oracle":"path","build":{"recipe":"none","arguments":[]},'
     '"start":{"runtime":"python","path":"app.py","arguments":[],"mode":"cli"},'
     '"steps":[{"type":"cli","arguments":["hello"]}]}. This is a minimal example, not evidence. '
-    "Recipes: none,pip,npm,composer,maven,gradle; none takes [], others one checkout-relative manifest/project path. "
+    "Recipes: none takes [], otherwise arguments[0] is a checkout-relative ecosystem manifest or project directory (a file uses "
+    "its parent): pip requirements*.txt uses install -r (e.g. requirements.txt), pyproject.toml/setup.py/setup.cfg uses "
+    "non-editable install of the directory with requirements.txt fallback when no build backend exists (e.g. pyproject.toml); npm uses "
+    "ci with package-lock.json else install in the directory (e.g. package.json); composer uses install --no-interaction in the "
+    "directory (e.g. composer.json); maven uses mvn -q -DskipTests package -f <pom> (e.g. pom.xml); gradle uses the project "
+    "wrapper and directory (e.g. build.gradle). "
     "Runtimes: python,node,php,java; start.path must be a tracked checkout entrypoint. "
     "HTTP mode requires port (1024-65535) and steps {type:http,method,path,headers?,body?}; "
     "CLI mode forbids port and uses {type:cli,arguments,stdin?}. Supply 1-32 ordered steps. "

tests/test_ax_batch.py

@@ -385,3 +385,27 @@ def test_executor_slot_deadline_has_reason(monkeypatch):
     monkeypatch.setattr(batch, "AX_SLOTS", SimpleNamespace(acquire=lambda **kw: False))
     with pytest.raises(batch.CapacityDeadline, match="no free workers available before deadline"):
         batch.SearchExecutorTask(object(), b"archive").__enter__()
+
+
+def test_search_preparation_failure_retains_command_record():
+    import time
+    from types import SimpleNamespace
+
+    from benchmarks.ax.batch import SearchExecutorTask
+    from openultrasast.search.task_storage import CommandFailure
+
+    task = object.__new__(SearchExecutorTask)
+    task.end = time.monotonic() + 60
+    task.client = SimpleNamespace(
+        submit=lambda *args, **kwargs: {
+            "status": "could_not_build",
+            "phase": "build",
+            "exit_code": 8,
+            "reason": "CommandFailure: exit code 8",
+            "stderr": "dependency missing token=hidden",
+        }
+    )
+    with pytest.raises(CommandFailure) as caught:
+        task.prepare({})
+    assert caught.value.phase == "build" and caught.value.exit_code == 8
+    assert "dependency missing" in caught.value.stderr and "hidden" not in caught.value.stderr

tests/test_search_demo.py

@@ -71,11 +71,10 @@ def test_directory_build_recipe_accepts_checkout_root(recipe):
 
 
 @pytest.mark.parametrize("recipe", ["pip", "maven"])
-def test_manifest_recipe_requires_file_path(recipe):
+def test_manifest_recipe_accepts_project_directory(recipe):
     value = document()
     value["build"] = {"recipe": recipe, "arguments": ["."]}
-    with pytest.raises(ValueError):
-        validate_demo(value)
+    assert validate_demo(value) == value
 
 
 @pytest.mark.parametrize("oracle", [None, "injection", "unknown", [], 1])
@@ -111,3 +110,16 @@ def test_family_oracle_contract(family, allowed):
         else:
             with pytest.raises(ValueError, match="not allowed"):
                 validate_oracle(family, oracle)
+
+
+@pytest.mark.parametrize(
+    "recipe,manifest",
+    [("pip", "package.json"), ("npm", "requirements.txt"), ("composer", "pom.xml"), ("maven", "composer.json"), ("gradle", "setup.py")],
+)
+@pytest.mark.parametrize("directory", [False, True])
+def test_recipe_rejects_other_ecosystem(tmp_path, recipe, manifest, directory):
+    (tmp_path / manifest).write_text("{}")
+    value = document()
+    value["build"] = {"recipe": recipe, "arguments": ["." if directory else manifest]}
+    with pytest.raises(ValueError, match="ecosystem"):
+        validate_demo(value, checkout=tmp_path)

tests/test_search_pilot_run.py

@@ -480,3 +480,23 @@ def test_injection_pilot_uses_demo_choice_without_manifest_subtype(tmp_path, ora
     (tmp_path / "demo/demo.json").write_bytes((demos["real"] / "demo.json").read_bytes())
     result = pilot.verification()
     assert result.outcome == "demonstrated", result
+
+
+@pytest.mark.parametrize("phase", ["build_export", "verify/could_not_run"])
+def test_pilot_retains_command_diagnostics(tmp_path, phase):
+    from openultrasast.search.task_storage import CommandFailure
+    from openultrasast.search.verify import SideRecord
+
+    pilot = make_pilot(tmp_path)
+    stderr = "\n".join(f"line-{i}" for i in range(30)) + "\ntoken=hidden https://example.test/?key=hidden"
+    if phase == "build_export":
+        pilot.failure(phase, CommandFailure("build", 7, stderr))
+    else:
+        side = SideRecord("could_not_run", (), 0, "start failed", phase="start", exit_code=7, stderr=stderr)
+        pilot.failure(phase, side.reason, command=side)
+    failure = pilot.record["failures"][-1]
+    assert failure["exit_code"] == 7
+    assert failure["command_phase"] == ("build" if phase == "build_export" else "start")
+    assert "line-29" in failure["stderr"] and "line-0\n" not in failure["stderr"]
+    assert "hidden" not in failure["stderr"] and "https://" not in failure["stderr"]
+    assert len(failure["stderr"]) <= 1500

tests/test_search_task_storage.py

@@ -142,7 +142,7 @@ def test_preparation_builds_before_export_and_exports_products(workspace, monkey
     blobs, commands = [], []
     monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
 
-    def build(argv, timeout, limits):
+    def build(argv, timeout, limits, **kwargs):
         commands.append(argv)
         products = Path(argv[argv.index("--target") + 1])
         assert products.is_relative_to(workspace)
@@ -286,14 +286,19 @@ def test_preparation_with_many_host_processes(workspace, monkeypatch):
 
 @pytest.mark.parametrize("memory_bytes", [None, 256 * 1024**2, 5 * 1024**3])
 def test_prepare_build_virtual_memory(workspace, monkeypatch, memory_bytes):
-    from openultrasast.search.demo import BUILD_RECIPES
 
     repo = workspace / "checkout"
     repo.mkdir()
     source = "import mmap; region=mmap.mmap(-1,512*1024**2); print(len(region))"
     (repo / "app.py").write_text(source)
     print("input_bytes=", len((repo / "app.py").read_bytes()))
-    monkeypatch.setitem(BUILD_RECIPES, "npm", (sys.executable, "-c", source))
+    (repo / "package.json").write_text("{}")
+    fake_bin = workspace / "bin"
+    fake_bin.mkdir()
+    tool = fake_bin / "npm"
+    tool.write_text("#!" + sys.executable + "\n" + source)
+    tool.chmod(0o755)
+    monkeypatch.setitem(executor.SAFE_ENV, "PATH", str(fake_bin) + ":/usr/bin:/bin")
     app = executor.InProcessExecutor(repo, task_boundary=True)
     app.prepared_put_url = "https://files.example/output"
     blobs = []
@@ -317,3 +322,101 @@ def test_verifier_app_virtual_memory(workspace):
     demo.write_text(json.dumps(spec()["demo"]))
     result = verify_side_task(Side(repo), demo, "path")
     assert result.outcome == "observed", result
+
+
+@pytest.mark.parametrize(
+    "recipe,manifest,extra,expected",
+    [
+        ("pip", "pyproject.toml", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "PROJECT"]),
+        ("pip", "requirements.txt", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "MANIFEST"]),
+        (
+            "pip",
+            "pyproject.toml",
+            {"requirements.txt": ""},
+            ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "REQUIREMENTS"],
+        ),
+        ("npm", "package.json", {}, ["install", "--ignore-scripts"]),
+        ("npm", "package.json", {"package-lock.json": "{}"}, ["ci", "--ignore-scripts"]),
+        ("composer", "composer.json", {}, ["install", "--no-interaction", "--no-plugins", "--no-scripts"]),
+        ("maven", "pom.xml", {}, ["-q", "-DskipTests", "package", "-f", "MANIFEST"]),
+    ],
+)
+@pytest.mark.parametrize("directory", [False, True])
+def test_recipe_command_with_real_fake_tool(workspace, monkeypatch, recipe, manifest, extra, expected, directory):
+    repo = workspace / "checkout"
+    project = repo / "project"
+    project.mkdir(parents=True)
+    (repo / "app.py").write_text("print('ready')")
+    contents = '[build-system]\nbuild-backend="setuptools.build_meta"\n' if manifest == "pyproject.toml" and not extra else ""
+    (project / manifest).write_text(contents)
+    for name, content in extra.items():
+        (project / name).write_text(content)
+    tools = workspace / "bin"
+    tools.mkdir()
+    tool = tools / {"pip": "python3", "maven": "mvn"}.get(recipe, recipe)
+    tool.write_text(
+        "#!" + sys.executable + "\nimport json, os, sys\nfrom pathlib import Path\n"
+        "Path('invocation.json').write_text(json.dumps([sys.argv[1:], os.getcwd()]))\n"
+    )
+    tool.chmod(0o755)
+    monkeypatch.setitem(executor.SAFE_ENV, "PATH", str(tools) + ":/usr/bin:/bin")
+    monkeypatch.setattr(executor.URLTransport, "put", lambda *args: None)
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    request = spec()
+    request["demo"]["build"] = {"recipe": recipe, "arguments": ["project" if directory else "project/" + manifest]}
+    result = app.submit("prepare_verification", request)
+    assert result["status"] == "ok", result
+    argv, cwd = json.loads((project / "invocation.json").read_text())
+    assert cwd == str(project)
+    replacements = {"PROJECT": str(project), "MANIFEST": str(project / manifest), "REQUIREMENTS": str(project / "requirements.txt")}
+    if "--target" in argv:
+        packages = argv[argv.index("--target") + 1]
+        assert Path(packages).name == "packages" and Path(packages).is_relative_to(workspace)
+        replacements["PACKAGES"] = packages
+    assert argv == [replacements.get(arg, arg) for arg in expected]
+
+
+def test_failed_build_preserves_sanitized_tail(workspace, monkeypatch):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "gradlew").write_text(
+        "for i in $(seq 1 30); do echo line-$i >&2; done\necho 'https://example.test/?token=secret token=hidden' >&2\nexit 7\n"
+    )
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    request = spec()
+    request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]}
+    result = app.submit("prepare_verification", request)
+    assert result["status"] == "could_not_build" and result["exit_code"] == 7
+    assert result["phase"] == "build"
+    assert "line-30" in result["stderr"] and "line-1\n" not in result["stderr"]
+    assert "hidden" not in result["stderr"] and "https://" not in result["stderr"]
+    assert len(result["stderr"]) <= 1500 and len(result["stderr"].splitlines()) <= 20
+
+
+@pytest.mark.parametrize("mode", ["cli", "http"])
+def test_verifier_start_failure_has_diagnostics(workspace, mode):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "app.py").write_text(
+        "import sys\nprint('start failed token=hidden https://example.test/?secret=x', file=sys.stderr)\nsys.exit(9)"
+    )
+    demo = workspace / "demo.json"
+    value = spec()["demo"]
+    value["start"]["mode"] = mode
+    if mode == "http":
+        value["start"]["port"] = 19001
+        value["steps"] = [{"type": "http", "method": "GET", "path": "/"}]
+    demo.write_text(json.dumps(value))
+    result = verify_side_task(Side(repo), demo, "path")
+    assert result.outcome == "could_not_run", result
+    assert result.phase == "start" and result.exit_code == 9
+    assert "start failed" in result.stderr and "hidden" not in result.stderr and "https://" not in result.stderr
+
+
+@pytest.mark.parametrize("secret", ['token="hidden"', '"password": "hidden"', "Bearer hidden", "ghp_hidden", "npm_hidden"])
+def test_command_failure_sanitizes_tokens_and_caps_tail(secret):
+    failure = task_storage.CommandFailure("build", 1, "old\n" * 50 + "x" * 2000 + "\n" + secret)
+    assert "hidden" not in failure.stderr and "old" not in failure.stderr
+    assert len(failure.stderr) <= 1500