search-with-proof: ecosystem-aware build recipes and stderr in build failures

The agent passes a manifest file or project directory; pip uses -r for
requirements files and the directory for pyproject/setup, npm ci/install runs in
the directory, composer/maven/gradle likewise. Build and verify failures carry
phase, exit code and a sanitised stderr tail.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/ax/batch.py

@@ -724,7 +724,11 @@ class SearchExecutorTask:
         remaining = min(300, self.end - time.monotonic())
         result = self.client.submit("prepare_verification", spec, timeout_seconds=remaining)
         if result.get("status") != "ok":
-            raise ValueError(result.get("reason", "could_not_build"))
+            from openultrasast.search.task_storage import CommandFailure
+
+            raise CommandFailure(
+                result.get("phase", "build"), result.get("exit_code"), result.get("stderr", ""), result.get("reason", "could_not_build")
+            )
         stored = bounded_read(lambda: self.lane.store._get(self.keys[3]), self.end - time.monotonic())
         if stored is None or hashlib.sha256(stored[0]).hexdigest() != result.get("sha256"):
             raise ValueError("executor archive missing or digest mismatch")

benchmarks/ax/search_verify.py

@@ -110,4 +110,7 @@ class AXSideDispatcher:
             max(r.scratch_peak_bytes for r in records),
             isolation_mode="task-boundary",
             chromium_no_sandbox=any(r.chromium_no_sandbox for r in records),
+            phase=last.phase,
+            exit_code=last.exit_code,
+            stderr=last.stderr,
         )

benchmarks/search/pilot_run.py

@@ -123,7 +123,7 @@ class Pilot:
         self.sides = None
         self.verifications = []
 
-    def failure(self, phase, reason):
+    def failure(self, phase, reason, command=None):
         private = tuple(str(v) for k, v in self.row.items() if k in {"repo", "file", "function", "vulnerable", "fixed"})
         private += (str(self.root), self.row["repo"].removeprefix("https://github.com/"))
         if isinstance(reason, BaseException):
@@ -135,7 +135,15 @@ class Pilot:
             safe = kind + ": " + diagnostic(message, maximum=max(0, 298 - len(kind)), private=private)
         else:
             safe = diagnostic(reason, private=private)
-        self.record["failures"].append(dict(phase=phase, reason=safe))
+        entry = dict(phase=phase, reason=safe)
+        command = command if command is not None else reason
+        if hasattr(command, "exit_code"):
+            entry.update(
+                command_phase=diagnostic(command.phase),
+                exit_code=command.exit_code,
+                stderr=diagnostic(command.stderr, maximum=1500, private=private),
+            )
+        self.record["failures"].append(entry)
 
     @contextmanager
     def executor(self, revision, deadline=900):
@@ -300,7 +308,7 @@ class Pilot:
 
                 for side in result.sides:
                     if side.outcome != "observed":
-                        self.failure("verify/" + side.outcome, "side_verification_failed")
+                        self.failure("verify/" + side.outcome, side.reason, command=side)
                 if result.outcome != "demonstrated":
                     self.failure("verify", "no_consistent_differential" if result.outcome == "inconclusive" else result.outcome)
                 end = result.outcome

src/openultrasast/search/demo.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import json
 import re
+import tomllib
 from pathlib import Path, PurePosixPath
 from typing import Any, cast
 
@@ -145,6 +146,75 @@ def checkout_path(value: Any) -> str:
     return value
 
 
+_MANIFESTS = {
+    "pip": ("pyproject.toml", "setup.py", "setup.cfg"),
+    "npm": ("package.json", "package-lock.json"),
+    "composer": ("composer.json", "composer.lock"),
+    "maven": ("pom.xml",),
+    "gradle": ("build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts", "gradlew"),
+}
+
+
+def build_project(recipe: str, path: Path, checkout: Path) -> Path:
+    """Validate the recipe ecosystem against actual checkout files."""
+
+    def manifest(candidate: Path) -> bool:
+        return candidate.name in _MANIFESTS[recipe] or (recipe == "pip" and candidate.match("requirements*.txt"))
+
+    def contained(candidate: Path) -> bool:
+        return candidate.resolve().is_relative_to(checkout.resolve())
+
+    if not contained(path):
+        raise ValueError("recipe input outside checkout")
+    if path.is_file():
+        if not manifest(path):
+            raise ValueError("recipe input is not an ecosystem manifest")
+        return path.parent
+    if path.is_dir() and any(p.is_file() and contained(p) and manifest(p) for p in path.iterdir()):
+        return path
+    raise ValueError("recipe input must be an ecosystem manifest or project directory")
+
+
+def preparation_command(recipe: str, path: Path, checkout: Path, products: Path) -> tuple[list[str], Path]:
+    project = build_project(recipe, path, checkout)
+    if recipe == "pip":
+        requirements = path if path.is_file() and path.match("requirements*.txt") else None
+        pyproject = project / "pyproject.toml"
+        backend = False
+        if requirements is None and pyproject.is_file():
+            if not pyproject.resolve().is_relative_to(checkout.resolve()):
+                raise ValueError("recipe input outside checkout")
+            backend = bool(tomllib.loads(pyproject.read_text()).get("build-system", {}).get("build-backend"))
+        if requirements is None and not backend and not (project / "setup.py").is_file():
+            fallback = project / "requirements.txt"
+            if fallback.is_file():
+                requirements = fallback
+        target = requirements or project
+        if not target.resolve().is_relative_to(checkout.resolve()):
+            raise ValueError("recipe input outside checkout")
+        return [
+            "python3",
+            "-I",
+            "-m",
+            "pip",
+            "install",
+            "--no-deps",
+            "--target",
+            str(products / "packages"),
+            *(["-r", str(target)] if requirements else [str(target)]),
+        ], project
+    if recipe == "npm":
+        return ["npm", "ci" if (project / "package-lock.json").is_file() else "install", "--ignore-scripts"], project
+    if recipe == "composer":
+        return ["composer", "install", "--no-interaction", "--no-plugins", "--no-scripts"], project
+    if recipe == "maven":
+        return ["mvn", "-q", "-DskipTests", "package", "-f", str(project / "pom.xml")], project
+    wrapper = project / "gradlew"
+    if not wrapper.is_file() or not wrapper.resolve().is_relative_to(checkout.resolve()):
+        raise ValueError("missing or external Gradle wrapper")
+    return ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(project), "assemble"], project
+
+
 def _text(value: Any, names: set[str]) -> str:
     if not isinstance(value, str) or "\0" in value or len(value.encode()) > MAX_VALUE_BYTES:
         raise ValueError("invalid or oversized value")
@@ -163,7 +233,7 @@ def _arguments(value: Any, names: set[str]) -> None:
         _text(arg, names)
 
 
-def validate_demo(value: Any) -> dict[str, Any]:
+def validate_demo(value: Any, *, checkout: Path | None = None) -> dict[str, Any]:
     try:
         encoded = json.dumps(value, allow_nan=False).encode()
     except (ValueError, TypeError, RecursionError) as exc:
@@ -186,8 +256,10 @@ def validate_demo(value: Any) -> dict[str, Any]:
     else:
         if len(args) != 1:
             raise ValueError("recipe requires one checkout path")
-        if not (recipe in ("npm", "composer", "gradle") and args[0] == "."):
+        if args[0] != ".":
             checkout_path(args[0])
+        if checkout is not None:
+            build_project(recipe, checkout / args[0], checkout)
     _keys(start, {"runtime", "path", "arguments", "mode"}, {"port"})
     if not isinstance(start["runtime"], str) or start["runtime"] not in RUNTIMES:
         raise ValueError("unknown runtime")

src/openultrasast/search/executor.py

@@ -269,12 +269,12 @@ class InProcessExecutor:
             "isolation_mode": self.isolation_mode,
             "exit_code": result.exit_code,
             "stdout": result.stdout[:16384],
-            "stderr": task_storage.diagnostic(result.stderr, maximum=4000),
+            "stderr": task_storage.diagnostic(result.stderr, maximum=1500),
             "timed_out": result.timed_out,
             "truncated": len(result.stdout) > 16384 or len(result.stderr) > 16384,
         }
 
-    def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]:
+    def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any], *, cwd: Path | None = None) -> dict[str, Any]:
         process_limit = None
         try:
             uid = os.getuid()
@@ -311,7 +311,7 @@ class InProcessExecutor:
         self.phase = "spawn"
         with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
             proc = subprocess.Popen(
-                argv, cwd=self.repo, env=clean_environment(), stdout=out, stderr=err, start_new_session=True, preexec_fn=constrain
+                argv, cwd=cwd or self.repo, env=clean_environment(), stdout=out, stderr=err, start_new_session=True, preexec_fn=constrain
             )
             self.phase = "run"
             timed_out = False
@@ -358,7 +358,7 @@ class InProcessExecutor:
                 proc.wait()
             out.seek(0)
             err.seek(max(0, os.fstat(err.fileno()).st_size - 16384))
-            stderr = task_storage.diagnostic(err.read().decode(errors="replace"), maximum=4000)
+            stderr = task_storage.diagnostic(err.read().decode(errors="replace"), maximum=1500)
             phase = "scratch guard" if disk_exceeded else "timeout" if timed_out else self.phase
             if disk_exceeded:
                 self.stopped = True
@@ -389,14 +389,14 @@ class InProcessExecutor:
         """Build and export through the host-scoped output URL, never a model URL."""
         import shutil
 
-        from .demo import BUILD_RECIPES, validate_demo
+        from .demo import preparation_command, validate_demo
         from .verify import _tree
         from .verify_task import pack_checkout, validate_spec
 
         if not self.task_boundary or not self.prepared_put_url:
             raise ValueError("preparation requires an executor task output")
         spec = copy.deepcopy(spec)
-        validate_demo(spec["demo"])
+        validate_demo(spec["demo"], checkout=self.repo)
         recipe = spec["demo"]["build"]
         spec["demo"]["build"] = {"recipe": "none", "arguments": []}
         validate_spec(spec)
@@ -406,22 +406,16 @@ class InProcessExecutor:
             products.mkdir()
             if recipe["recipe"] != "none":
                 path = self._path(recipe["arguments"][0])
-                argv = [
+                argv, project = preparation_command(recipe["recipe"], path, self.repo, products)
-                    arg.replace("/scratch", str(products))
+                self.phase = "build"
-                    for arg in BUILD_RECIPES[recipe["recipe"]]
+                built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES, **limits}, cwd=project)
-                    if arg not in {"--offline", "--no-index"}
-                ]
-                argv.append(str(path))
-                if recipe["recipe"] == "maven":
-                    argv += ["package", "-DskipTests"]
-                elif recipe["recipe"] == "gradle":
-                    # The project pins its Gradle version through its wrapper.
-                    # Invoke with sh because checkout archives may lose execute bits.
-                    wrapper = self._path(str(path.relative_to(self.repo) / "gradlew"))
-                    argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"]
-                built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES, **limits})
                 if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
-                    return {**built, "status": "could_not_build"}
+                    return {
+                        **built,
+                        "status": "could_not_build",
+                        "phase": "build",
+                        "stderr": task_storage.diagnostic(built.get("stderr", ""), maximum=1500),
+                    }
             # Validate internal generated links before materializing them as
             # regular archive members. No host/external files may be exported.
             _tree(self.repo, generated_links=True)

src/openultrasast/search/probe.py

@@ -15,16 +15,16 @@ import subprocess
 import tempfile
 import time
 import urllib.request
-from dataclasses import asdict
+from dataclasses import asdict, replace
 from importlib.resources import files
 from pathlib import Path
 from typing import Any
 
 from . import _sandbox
-from .demo import FAMILY_ORACLES
+from .demo import FAMILY_ORACLES, load_demo
 from .oracles import BrowserExecutor
 from .task_storage import exception_reason
-from .verify import Side, verify
+from .verify import Side, VerificationRecord, verify, verify_side_task
 
 FAMILIES = ("sql", "path", "command", "ssrf", "xss")
 PAYLOADS = {
@@ -105,6 +105,55 @@ def namespace_facts() -> dict[str, Any]:
     return facts
 
 
+def _verify_task_pair(sides: tuple[Side, Side], demo: Path, family: str) -> VerificationRecord:
+    """Exercise the task executor on packaged toys only, not arbitrary checkouts.
+
+    Repetitions get fresh scratch, but share this probe task's outer boundary.
+    This is an instrument self-test, not a fresh-task dispatcher for evidence.
+    """
+    started = time.monotonic()
+    records = []
+    try:
+        load_demo(demo)
+    except (OSError, ValueError) as exc:
+        return VerificationRecord(
+            "inconclusive",
+            (),
+            time.monotonic() - started,
+            "invalid declarative demo: " + exception_reason(exc),
+            isolation_mode="task-boundary",
+        )
+    search_family = next(f for f, kinds in FAMILY_ORACLES.items() if family in kinds)
+    for side in sides:
+        runs = [verify_side_task(side, demo, search_family) for _ in range(3)]
+        first = next((run for run in runs if run.outcome != "observed"), runs[0])
+        records.append(
+            replace(
+                first,
+                runs=tuple(observation for run in runs for observation in run.runs),
+                elapsed_seconds=sum(run.elapsed_seconds for run in runs),
+                build_seconds=sum(run.build_seconds for run in runs),
+                ready_seconds=sum(run.ready_seconds for run in runs),
+                run_seconds=sum(run.run_seconds for run in runs),
+                scratch_peak_bytes=max(run.scratch_peak_bytes for run in runs),
+                reason=first.reason if first.outcome != "observed" else "three probe task executions completed",
+            )
+        )
+    demonstrated = (
+        all(side.outcome == "observed" and len(side.runs) == 3 for side in records)
+        and all(run.observed for run in records[0].runs)
+        and not any(run.observed for run in records[1].runs)
+    )
+    return VerificationRecord(
+        "demonstrated" if demonstrated else "inconclusive",
+        tuple(records),
+        time.monotonic() - started,
+        "owned effect in 3/3 affected runs and 0/3 safe runs" if demonstrated else "no consistent differential effect",
+        isolation_mode="task-boundary",
+        chromium_no_sandbox=any(side.chromium_no_sandbox for side in records),
+    )
+
+
 def probe(root: Path) -> dict[str, Any]:
     record: dict[str, Any] = {
         "status": "ok",
@@ -114,26 +163,54 @@ def probe(root: Path) -> dict[str, Any]:
         "isolation_check": None,
         "isolation_mode": _sandbox.isolation_mode(),
     }
+    namespaces = record["namespaces"]
+    if namespaces and all(namespaces[name]["exit_code"] != 0 for name in ("user", "pid", "net", "ipc", "uts", "mount")):
+        record["isolation_mode"] = "task-boundary"
+    task_boundary = record["isolation_mode"] == "task-boundary"
+    if task_boundary:
+        # Real verification still needs fresh side tasks; only these packaged
+        # self-test applications share the probe task's boundary.
+        record["requires_fresh_side_tasks"] = True
     start = time.monotonic()
     try:
-        _sandbox.isolation_check()
+        if task_boundary:
-        record["isolation_check"] = {"status": "ok", "input_bytes": len(b"isolation-ready\n")}
+            record["isolation_check"] = {"status": "skipped", "reason": "task-boundary"}
+        else:
+            _sandbox.isolation_check()
+            record["isolation_check"] = {"status": "ok", "input_bytes": len(b"isolation-ready\n")}
         for family in FAMILIES:
             sides, demos = materialise(root / family, family)
             results: dict[str, Any] = {}
             record["oracles"][family] = results
             for name, demo in demos.items():
-                result = verify(
+                search_family = next(f for f, kinds in FAMILY_ORACLES.items() if family in kinds)
-                    *sides,
+                if task_boundary:
-                    demo,
+                    tick = time.monotonic()
-                    next(f for f, kinds in FAMILY_ORACLES.items() if family in kinds),
+                    try:
-                    browser=BrowserExecutor() if family == "xss" else None,
+                        result = _verify_task_pair(sides, demo, family)
-                )
+                    except Exception as exc:
+                        result = VerificationRecord(
+                            "inconclusive", (), time.monotonic() - tick, exception_reason(exc), isolation_mode="task-boundary"
+                        )
+                else:
+                    result = verify(
+                        *sides,
+                        demo,
+                        search_family,
+                        oracle=family,
+                        browser=BrowserExecutor() if family == "xss" else None,
+                    )
                 results[name] = asdict(result)
+                if task_boundary and (result.outcome == "demonstrated") != (name == "real") and record["status"] == "ok":
+                    expected = "demonstrated" if name == "real" else "not demonstrated"
+                    record.update(
+                        status="instrument_failure",
+                        exception=f"{family}/{name}: expected {expected}, got {result.outcome}",
+                    )
                 # SSRF/browser need additional namespace capabilities beyond the
                 # generic preflight. These failures must not look like oracle misses.
                 for side in result.sides:
-                    if side.reason.startswith("verification unavailable or refused:"):
+                    if not task_boundary and side.reason.startswith("verification unavailable or refused:"):
                         raise _sandbox.IsolationUnavailable(f"{family}/{name}: {side.reason}")
     except Exception as exc:
         record.update(status="instrument_failure", exception=exception_reason(exc))

src/openultrasast/search/task_storage.py

@@ -106,8 +106,12 @@ def diagnostic(value: object, *, maximum: int = 300, private: tuple[str, ...] =
     text = re.sub(r"(?i)bearer\s+[^\s,;\"']+", "Bearer [redacted]", text)
     text = re.sub(r"(?i)(?:https?|s3)://[^\s<>\"']+", "[redacted-url]", text)
     text = re.sub(
-        r"(?i)([\w-]*(?:authorization|signature|credential|token|secret|password|key))\s*[:=]\s*[^\s,;&\"']+", r"\1=[redacted]", text
+        r"(?i)([\w-]*(?:authorization|signature|credential|token|secret|password|key))[\"']?\s*[:=]\s*[\"']?[^\s,;&\"']+",
+        r"\1=[redacted]",
+        text,
     )
+    text = re.sub(r"\b(?:gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+|sk-[A-Za-z0-9_-]+|npm_[A-Za-z0-9]+)\b", "[redacted]", text)
+    text = re.sub(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b", "[redacted]", text)
     for name in sorted(set(private), key=len, reverse=True):
         if name:
             text = text.replace(name, "[repository]")
@@ -125,3 +129,14 @@ def exception_reason(exc: BaseException, *, private: tuple[str, ...] = ()) -> st
         half = max(0, (maximum - 2) // 2)
         return prefix + diagnostic(original, maximum=half, private=private) + "; " + diagnostic(cleanup, maximum=half, private=private)
     return prefix + diagnostic(exc, maximum=maximum, private=private)
+
+
+class CommandFailure(ValueError):
+    """Transport a bounded command diagnostic without dropping stderr."""
+
+    def __init__(self, phase: str, exit_code: int | None, stderr: str, reason: str = "") -> None:
+        self.phase = phase
+        self.exit_code = exit_code
+        self.stderr = diagnostic(stderr, maximum=1500)
+        summary = diagnostic(reason) if reason else f"{phase}: exit code {exit_code}"
+        super().__init__(summary + (": " + self.stderr if self.stderr else ""))

src/openultrasast/search/verify.py

@@ -30,7 +30,7 @@ from . import _sandbox, task_storage
 from .budget import SearchBudget
 from .demo import BUILD_RECIPES, FAMILY_ORACLES, RUNTIMES, capture_output, load_demo, substitute, validate_oracle
 from .oracles import BrowserExecutor, Canary, Oracle, oracle_for
-from .task_storage import exception_reason
+from .task_storage import CommandFailure, diagnostic, exception_reason
 
 
 @dataclass(frozen=True)
@@ -62,6 +62,9 @@ class SideRecord:
     scratch_peak_bytes: int = 0
     isolation_mode: str = field(default_factory=_sandbox.isolation_mode)
     chromium_no_sandbox: bool = False
+    phase: str = ""
+    exit_code: int | None = None
+    stderr: str = ""
 
 
 @dataclass(frozen=True)
@@ -133,18 +136,19 @@ def _tree(root: Path, *, generated_links: bool = False) -> str:
 # Executed as verifier-owned code inside the same network sandbox as the app.
 # Redirects are refused: only the configured loopback application is reachable.
 _HTTP_DRIVER = r"""
-import http.client, importlib.util, json, os, socket, subprocess, sys, time
+import http.client, importlib.util, json, os, socket, subprocess, sys, tempfile, time
 spec = importlib.util.spec_from_file_location('demo', '/verifier-demo.py')
 demo = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(demo)
 config = json.load(sys.stdin)
 schema = config['schema']
-app = subprocess.Popen(config['command'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+app_errors = tempfile.TemporaryFile()
+app = subprocess.Popen(config['command'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=app_errors)
 try:
     until = time.monotonic() + config['timeout'] / 2
     while True:
         if app.poll() is not None:
-            raise RuntimeError('HTTP application exited during readiness')
+            raise SystemExit(app.returncode or 1)
         try:
             connection = socket.create_connection(('127.0.0.1', schema['start']['port']), timeout=0.1)
             connection.close()
@@ -183,6 +187,9 @@ finally:
     except subprocess.TimeoutExpired:
         app.kill()
         app.wait()
+    app_errors.seek(max(0, app_errors.seek(0, 2) - 16384))
+    sys.stderr.write(app_errors.read().decode(errors="replace"))
+    app_errors.close()
 """
 
 
@@ -223,7 +230,12 @@ def _http_steps(
             acknowledged = True
         result = future.result()
     if not acknowledged or result.exit_code or result.timed_out:
-        raise ValueError("HTTP application/steps failed or readiness not acknowledged: " + result.stderr[-2000:])
+        raise CommandFailure(
+            "start" if not acknowledged else "run",
+            result.exit_code,
+            result.stderr,
+            "HTTP application/steps failed or readiness not acknowledged",
+        )
     outputs = json.loads(result.stdout)
     if not isinstance(outputs, list) or len(outputs) != len(schema["steps"]) or not all(isinstance(value, str) for value in outputs):
         raise ValueError("invalid HTTP driver output")
@@ -249,18 +261,21 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
             if root is not None:
                 disk_peak = max(disk_peak, sum(p.lstat().st_blocks * 512 for p in root.rglob("*")))
 
-    def record(outcome: str, reason: str) -> SideRecord:
+    def record(outcome: str, reason: str, phase: str = "", result: Any = None) -> SideRecord:
         return SideRecord(
             outcome,
             tuple(observations),
             time.monotonic() - started,
-            reason,
+            diagnostic(reason, maximum=1500),
             timings["build"],
             timings["ready"],
             timings["run"],
             disk_peak,
             isolation_mode="task-boundary" if task_boundary else _sandbox.isolation_mode(),
             chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)),
+            phase=phase,
+            exit_code=result.exit_code if result is not None else None,
+            stderr=diagnostic(result.stderr, maximum=1500) if result is not None else "",
         )
 
     try:
@@ -305,7 +320,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                         command += ("assemble",)
                     compiled = timed("build", runner, job(command), scratch=products, mounts=mounts, writable_checkout=True)
                     if compiled.exit_code or compiled.timed_out:
-                        return record("could_not_build", "build recipe failed: " + compiled.stderr[-2000:])
+                        return record("could_not_build", "build recipe failed", "build", compiled)
                 if tuple(_tree(p) for p in immutable) != before:
                     return record("inconclusive", "build changed immutable inputs")
                 product_hash = _tree(products, generated_links=True)
@@ -354,7 +369,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                             namespace_pid=oracle.namespace_pid,
                         )
                         if ready.exit_code or ready.timed_out:
-                            return record("could_not_run", "app readiness failed: " + ready.stderr[-2000:])
+                            return record("could_not_run", "app readiness failed", "start", ready)
                     if family == "ssrf" and oracle.observe()[0]:
                         return record("inconclusive", "readiness triggered the SSRF oracle")
                     # Readiness cannot supply the attack's evidence or leave a command marker.
@@ -401,7 +416,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                             if result.timed_out:
                                 return record("inconclusive", "app attack timed out")
                             if result.exit_code:
-                                return record("could_not_run", "app invocation failed")
+                                return record("could_not_run", "app invocation failed", "run", result)
                             timed("run", oracle.capture, result.stdout)
                             capture_output(step, result.stdout, captures, canary.nonce)
                     observed, evidence = timed("run", oracle.observe)
@@ -416,6 +431,8 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                 finally:
                     oracle.close()
         return record("observed", "fresh task run completed" if task_boundary else "three fresh runs completed")
+    except CommandFailure as exc:
+        return record("could_not_run", str(exc), exc.phase, exc)
     except task_storage.ScratchLimit as exc:
         return record("could_not_build", exception_reason(exc))
     except (OSError, ValueError, AssertionError, KeyError, IndexError, TypeError) as exc:
@@ -437,10 +454,11 @@ def _run_task(
     the filesystem/network boundary. Its launch manifest must contain no keys.
     """
     translations = {"/workspace": str(job.repo_root), "/scratch": str(scratch), **{k: str(v) for k, v in (mounts or {}).items()}}
-    pattern = re.compile("|".join(re.escape(key) for key in sorted(translations, key=len, reverse=True)))
+    pattern = re.compile("(?:" + "|".join(re.escape(key) for key in sorted(translations, key=len, reverse=True)) + r")(?=/|$|['\"])")
 
     def translate(value: str) -> str:
-        # One substitution pass: host temp paths may themselves contain /scratch.
+        # One pass, at path-component boundaries: /fixture/fixture.db must
+        # translate its mount prefix without also rewriting the filename.
         return pattern.sub(lambda match: translations[match.group(0)], value)
 
     command = tuple(translate(arg) for arg in job.command)
@@ -471,7 +489,7 @@ def _run_task(
                 os.killpg(proc.pid, signal.SIGKILL)
             proc.wait()
         out.seek(0)
-        err.seek(0)
+        err.seek(max(0, os.fstat(err.fileno()).st_size - 16384))
         return SandboxResult(
             proc.returncode, out.read(1048576).decode(errors="replace"), err.read(1048576).decode(errors="replace"), expired
         )

src/openultrasast/search/worker.py

@@ -81,7 +81,12 @@ DEMO_GUIDE = (
     'Demo schema: {"oracle":"path","build":{"recipe":"none","arguments":[]},'
     '"start":{"runtime":"python","path":"app.py","arguments":[],"mode":"cli"},'
     '"steps":[{"type":"cli","arguments":["hello"]}]}. This is a minimal example, not evidence. '
-    "Recipes: none,pip,npm,composer,maven,gradle; none takes [], others one checkout-relative manifest/project path. "
+    "Recipes: none takes [], otherwise arguments[0] is a checkout-relative ecosystem manifest or project directory (a file uses "
+    "its parent): pip requirements*.txt uses install -r (e.g. requirements.txt), pyproject.toml/setup.py/setup.cfg uses "
+    "non-editable install of the directory with requirements.txt fallback when no build backend exists (e.g. pyproject.toml); npm uses "
+    "ci with package-lock.json else install in the directory (e.g. package.json); composer uses install --no-interaction in the "
+    "directory (e.g. composer.json); maven uses mvn -q -DskipTests package -f <pom> (e.g. pom.xml); gradle uses the project "
+    "wrapper and directory (e.g. build.gradle). "
     "Runtimes: python,node,php,java; start.path must be a tracked checkout entrypoint. "
     "HTTP mode requires port (1024-65535) and steps {type:http,method,path,headers?,body?}; "
     "CLI mode forbids port and uses {type:cli,arguments,stdin?}. Supply 1-32 ordered steps. "

tests/test_ax_batch.py

@@ -385,3 +385,27 @@ def test_executor_slot_deadline_has_reason(monkeypatch):
     monkeypatch.setattr(batch, "AX_SLOTS", SimpleNamespace(acquire=lambda **kw: False))
     with pytest.raises(batch.CapacityDeadline, match="no free workers available before deadline"):
         batch.SearchExecutorTask(object(), b"archive").__enter__()
+
+
+def test_search_preparation_failure_retains_command_record():
+    import time
+    from types import SimpleNamespace
+
+    from benchmarks.ax.batch import SearchExecutorTask
+    from openultrasast.search.task_storage import CommandFailure
+
+    task = object.__new__(SearchExecutorTask)
+    task.end = time.monotonic() + 60
+    task.client = SimpleNamespace(
+        submit=lambda *args, **kwargs: {
+            "status": "could_not_build",
+            "phase": "build",
+            "exit_code": 8,
+            "reason": "CommandFailure: exit code 8",
+            "stderr": "dependency missing token=hidden",
+        }
+    )
+    with pytest.raises(CommandFailure) as caught:
+        task.prepare({})
+    assert caught.value.phase == "build" and caught.value.exit_code == 8
+    assert "dependency missing" in caught.value.stderr and "hidden" not in caught.value.stderr

tests/test_search_demo.py

@@ -71,11 +71,10 @@ def test_directory_build_recipe_accepts_checkout_root(recipe):
 
 
 @pytest.mark.parametrize("recipe", ["pip", "maven"])
-def test_manifest_recipe_requires_file_path(recipe):
+def test_manifest_recipe_accepts_project_directory(recipe):
     value = document()
     value["build"] = {"recipe": recipe, "arguments": ["."]}
-    with pytest.raises(ValueError):
+    assert validate_demo(value) == value
-        validate_demo(value)
 
 
 @pytest.mark.parametrize("oracle", [None, "injection", "unknown", [], 1])
@@ -111,3 +110,16 @@ def test_family_oracle_contract(family, allowed):
         else:
             with pytest.raises(ValueError, match="not allowed"):
                 validate_oracle(family, oracle)
+
+
+@pytest.mark.parametrize(
+    "recipe,manifest",
+    [("pip", "package.json"), ("npm", "requirements.txt"), ("composer", "pom.xml"), ("maven", "composer.json"), ("gradle", "setup.py")],
+)
+@pytest.mark.parametrize("directory", [False, True])
+def test_recipe_rejects_other_ecosystem(tmp_path, recipe, manifest, directory):
+    (tmp_path / manifest).write_text("{}")
+    value = document()
+    value["build"] = {"recipe": recipe, "arguments": ["." if directory else manifest]}
+    with pytest.raises(ValueError, match="ecosystem"):
+        validate_demo(value, checkout=tmp_path)

tests/test_search_pilot_run.py

@@ -480,3 +480,23 @@ def test_injection_pilot_uses_demo_choice_without_manifest_subtype(tmp_path, ora
     (tmp_path / "demo/demo.json").write_bytes((demos["real"] / "demo.json").read_bytes())
     result = pilot.verification()
     assert result.outcome == "demonstrated", result
+
+
+@pytest.mark.parametrize("phase", ["build_export", "verify/could_not_run"])
+def test_pilot_retains_command_diagnostics(tmp_path, phase):
+    from openultrasast.search.task_storage import CommandFailure
+    from openultrasast.search.verify import SideRecord
+
+    pilot = make_pilot(tmp_path)
+    stderr = "\n".join(f"line-{i}" for i in range(30)) + "\ntoken=hidden https://example.test/?key=hidden"
+    if phase == "build_export":
+        pilot.failure(phase, CommandFailure("build", 7, stderr))
+    else:
+        side = SideRecord("could_not_run", (), 0, "start failed", phase="start", exit_code=7, stderr=stderr)
+        pilot.failure(phase, side.reason, command=side)
+    failure = pilot.record["failures"][-1]
+    assert failure["exit_code"] == 7
+    assert failure["command_phase"] == ("build" if phase == "build_export" else "start")
+    assert "line-29" in failure["stderr"] and "line-0\n" not in failure["stderr"]
+    assert "hidden" not in failure["stderr"] and "https://" not in failure["stderr"]
+    assert len(failure["stderr"]) <= 1500

tests/test_search_probe.py

@@ -2,12 +2,22 @@ import json
 import sys
 from pathlib import Path
 
+import pytest
+
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
 from openultrasast.search import probe
 from openultrasast.search._sandbox import IsolationUnavailable
 
 
+@pytest.fixture(autouse=True)
+def namespace_diagnostics(monkeypatch):
+    facts = {name: {"exit_code": 0} for name in ("user", "pid", "net", "ipc", "uts", "mount")}
+    monkeypatch.setattr(probe, "namespace_facts", lambda: facts)
+    monkeypatch.setattr(probe._sandbox, "isolation_mode", lambda: "userns")
+    return facts
+
+
 def test_isolation_failure_is_recorded_and_nonzero(monkeypatch, capsys):
     def fail(**kwargs):
         raise IsolationUnavailable("gVisor denied namespace")
@@ -118,3 +128,121 @@ def test_oracle_namespace_failure_is_instrument_failure(tmp_path, monkeypatch):
     assert record["status"] == "instrument_failure"
     assert "net namespace denied" in record["exception"]
     assert record["isolation_check"]["status"] == "ok"
+
+
+@pytest.mark.parametrize("execution", ["real", "stubbed"])
+def test_task_boundary_probe_executes_packaged_pairs(tmp_path, monkeypatch, namespace_diagnostics, execution):
+    if execution == "real":
+        import socket
+
+        try:
+            with socket.socket() as listener:
+                listener.bind(("127.0.0.1", 0))
+        except PermissionError:
+            pytest.skip("host policy prohibits localhost sockets required by SSRF and Chromium")
+    else:
+        from openultrasast.search.verify import RunObservation, SideRecord
+
+        def run(side, demo, family):
+            assert (side.checkout / "app.py").read_bytes()
+            observed = not (side.checkout / "fixed").exists()
+            return SideRecord(
+                "observed",
+                (RunObservation(observed, "stubbed owned effect", 0.5),),
+                0.7,
+                "stubbed execution",
+                0,
+                0.2,
+                0.5,
+                4096,
+                "task-boundary",
+                json.loads((demo / "demo.json").read_text())["oracle"] == "xss",
+            )
+
+        monkeypatch.setattr(probe, "verify_side_task", run)
+    monkeypatch.setattr(probe._sandbox, "isolation_mode", lambda: "task-boundary")
+    monkeypatch.setattr(probe, "image_facts", lambda: {})
+    monkeypatch.setattr(probe._sandbox, "isolation_check", lambda **kw: pytest.fail("task boundary must skip preflight"))
+    monkeypatch.setattr(probe, "verify", lambda *a, **kw: pytest.fail("must use task executor"))
+    record = probe.probe(tmp_path)
+    assert record["status"] == "ok", record
+    assert record["isolation_check"] == {"status": "skipped", "reason": "task-boundary"}
+    assert record["isolation_mode"] == "task-boundary"
+    assert record["namespaces"] == namespace_diagnostics
+    assert set(record["oracles"]) == set(probe.FAMILIES)
+    for family, demos in record["oracles"].items():
+        assert set(demos) == {"real", "observer_access", "revision_patch"}
+        for name, result in demos.items():
+            assert (result["outcome"] == "demonstrated") == (name == "real")
+            assert result["isolation_mode"] == "task-boundary"
+            assert result["elapsed_seconds"] > 0
+        real = demos["real"]
+        assert real["chromium_no_sandbox"] == (family == "xss")
+        for index, side in enumerate(real["sides"]):
+            assert len(side["runs"]) == 3
+            assert all(run["observed"] == (index == 0) for run in side["runs"])
+            assert side["isolation_mode"] == "task-boundary"
+            assert side["chromium_no_sandbox"] == (family == "xss")
+            assert side["build_seconds"] == 0
+            assert side["ready_seconds"] > 0
+            assert side["run_seconds"] > 0
+            assert side["scratch_peak_bytes"] > 0
+    assert record["wall_seconds"] > 0
+    assert record["peak_rss_kib"]["sum"] > 0
+    assert record["scratch_disk_bytes"] > 0
+    assert record["verification_scratch_peak_bytes"] > 0
+    assert record["scratch_filesystem_used_bytes"] > 0
+
+
+@pytest.mark.parametrize("family", ["sql", "path", "command"])
+def test_task_boundary_real_local_oracles(tmp_path, family):
+    sides, demos = probe.materialise(tmp_path / family, family)
+    result = probe._verify_task_pair(sides, demos["real"], family)
+    assert result.outcome == "demonstrated", result
+    assert all(len(side.runs) == 3 for side in result.sides)
+
+
+def test_task_boundary_xss_uses_browser_mode(tmp_path, monkeypatch):
+    from openultrasast.search.oracles import BrowserExecutor
+
+    calls = []
+
+    def capture(self, document, nonce):
+        assert self.task_boundary and self.no_sandbox
+        calls.append(document)
+        return '<script>alert("ousast-xss")</script>' in document
+
+    monkeypatch.setattr(BrowserExecutor, "__call__", capture)
+    sides, demos = probe.materialise(tmp_path / "xss", "xss")
+    result = probe._verify_task_pair(sides, demos["real"], "xss")
+    assert result.outcome == "demonstrated"
+    assert result.chromium_no_sandbox
+    assert all(side.chromium_no_sandbox for side in result.sides)
+    assert len(calls) == 6
+
+
+@pytest.mark.parametrize("mismatch", ["real", "observer_access", "revision_patch", "exception"])
+def test_task_boundary_mismatch_names_first_and_finishes(tmp_path, monkeypatch, mismatch):
+    from openultrasast.search.verify import VerificationRecord
+
+    # On AX the diagnostics select the boundary before the failed bwrap preflight.
+    facts = {name: {"exit_code": 1} for name in ("user", "pid", "net", "ipc", "uts", "mount")}
+    monkeypatch.setattr(probe, "namespace_facts", lambda: facts)
+    monkeypatch.setattr(probe, "image_facts", lambda: {})
+    monkeypatch.setattr(probe._sandbox, "isolation_check", lambda **kw: pytest.fail("must skip preflight"))
+
+    def result(sides, demo, family):
+        if mismatch == "exception":
+            raise OSError("executor unavailable")
+        demonstrated = demo.name == "real"
+        if demo.name == mismatch:
+            demonstrated = not demonstrated
+        return VerificationRecord("demonstrated" if demonstrated else "inconclusive", (), 1, "", "task-boundary")
+
+    monkeypatch.setattr(probe, "_verify_task_pair", result)
+    record = probe.probe(tmp_path)
+    assert record["status"] == "instrument_failure"
+    assert record["exception"].startswith(f"sql/{'real' if mismatch == 'exception' else mismatch}:")
+    assert record["namespaces"] == facts
+    assert record["isolation_check"] == {"status": "skipped", "reason": "task-boundary"}
+    assert sum(len(demos) for demos in record["oracles"].values()) == 15

tests/test_search_task_storage.py

@@ -142,7 +142,7 @@ def test_preparation_builds_before_export_and_exports_products(workspace, monkey
     blobs, commands = [], []
     monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
 
-    def build(argv, timeout, limits):
+    def build(argv, timeout, limits, **kwargs):
         commands.append(argv)
         products = Path(argv[argv.index("--target") + 1])
         assert products.is_relative_to(workspace)
@@ -286,14 +286,19 @@ def test_preparation_with_many_host_processes(workspace, monkeypatch):
 
 @pytest.mark.parametrize("memory_bytes", [None, 256 * 1024**2, 5 * 1024**3])
 def test_prepare_build_virtual_memory(workspace, monkeypatch, memory_bytes):
-    from openultrasast.search.demo import BUILD_RECIPES
 
     repo = workspace / "checkout"
     repo.mkdir()
     source = "import mmap; region=mmap.mmap(-1,512*1024**2); print(len(region))"
     (repo / "app.py").write_text(source)
     print("input_bytes=", len((repo / "app.py").read_bytes()))
-    monkeypatch.setitem(BUILD_RECIPES, "npm", (sys.executable, "-c", source))
+    (repo / "package.json").write_text("{}")
+    fake_bin = workspace / "bin"
+    fake_bin.mkdir()
+    tool = fake_bin / "npm"
+    tool.write_text("#!" + sys.executable + "\n" + source)
+    tool.chmod(0o755)
+    monkeypatch.setitem(executor.SAFE_ENV, "PATH", str(fake_bin) + ":/usr/bin:/bin")
     app = executor.InProcessExecutor(repo, task_boundary=True)
     app.prepared_put_url = "https://files.example/output"
     blobs = []
@@ -317,3 +322,101 @@ def test_verifier_app_virtual_memory(workspace):
     demo.write_text(json.dumps(spec()["demo"]))
     result = verify_side_task(Side(repo), demo, "path")
     assert result.outcome == "observed", result
+
+
+@pytest.mark.parametrize(
+    "recipe,manifest,extra,expected",
+    [
+        ("pip", "pyproject.toml", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "PROJECT"]),
+        ("pip", "requirements.txt", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "MANIFEST"]),
+        (
+            "pip",
+            "pyproject.toml",
+            {"requirements.txt": ""},
+            ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "REQUIREMENTS"],
+        ),
+        ("npm", "package.json", {}, ["install", "--ignore-scripts"]),
+        ("npm", "package.json", {"package-lock.json": "{}"}, ["ci", "--ignore-scripts"]),
+        ("composer", "composer.json", {}, ["install", "--no-interaction", "--no-plugins", "--no-scripts"]),
+        ("maven", "pom.xml", {}, ["-q", "-DskipTests", "package", "-f", "MANIFEST"]),
+    ],
+)
+@pytest.mark.parametrize("directory", [False, True])
+def test_recipe_command_with_real_fake_tool(workspace, monkeypatch, recipe, manifest, extra, expected, directory):
+    repo = workspace / "checkout"
+    project = repo / "project"
+    project.mkdir(parents=True)
+    (repo / "app.py").write_text("print('ready')")
+    contents = '[build-system]\nbuild-backend="setuptools.build_meta"\n' if manifest == "pyproject.toml" and not extra else ""
+    (project / manifest).write_text(contents)
+    for name, content in extra.items():
+        (project / name).write_text(content)
+    tools = workspace / "bin"
+    tools.mkdir()
+    tool = tools / {"pip": "python3", "maven": "mvn"}.get(recipe, recipe)
+    tool.write_text(
+        "#!" + sys.executable + "\nimport json, os, sys\nfrom pathlib import Path\n"
+        "Path('invocation.json').write_text(json.dumps([sys.argv[1:], os.getcwd()]))\n"
+    )
+    tool.chmod(0o755)
+    monkeypatch.setitem(executor.SAFE_ENV, "PATH", str(tools) + ":/usr/bin:/bin")
+    monkeypatch.setattr(executor.URLTransport, "put", lambda *args: None)
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    request = spec()
+    request["demo"]["build"] = {"recipe": recipe, "arguments": ["project" if directory else "project/" + manifest]}
+    result = app.submit("prepare_verification", request)
+    assert result["status"] == "ok", result
+    argv, cwd = json.loads((project / "invocation.json").read_text())
+    assert cwd == str(project)
+    replacements = {"PROJECT": str(project), "MANIFEST": str(project / manifest), "REQUIREMENTS": str(project / "requirements.txt")}
+    if "--target" in argv:
+        packages = argv[argv.index("--target") + 1]
+        assert Path(packages).name == "packages" and Path(packages).is_relative_to(workspace)
+        replacements["PACKAGES"] = packages
+    assert argv == [replacements.get(arg, arg) for arg in expected]
+
+
+def test_failed_build_preserves_sanitized_tail(workspace, monkeypatch):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "gradlew").write_text(
+        "for i in $(seq 1 30); do echo line-$i >&2; done\necho 'https://example.test/?token=secret token=hidden' >&2\nexit 7\n"
+    )
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    request = spec()
+    request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]}
+    result = app.submit("prepare_verification", request)
+    assert result["status"] == "could_not_build" and result["exit_code"] == 7
+    assert result["phase"] == "build"
+    assert "line-30" in result["stderr"] and "line-1\n" not in result["stderr"]
+    assert "hidden" not in result["stderr"] and "https://" not in result["stderr"]
+    assert len(result["stderr"]) <= 1500 and len(result["stderr"].splitlines()) <= 20
+
+
+@pytest.mark.parametrize("mode", ["cli", "http"])
+def test_verifier_start_failure_has_diagnostics(workspace, mode):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "app.py").write_text(
+        "import sys\nprint('start failed token=hidden https://example.test/?secret=x', file=sys.stderr)\nsys.exit(9)"
+    )
+    demo = workspace / "demo.json"
+    value = spec()["demo"]
+    value["start"]["mode"] = mode
+    if mode == "http":
+        value["start"]["port"] = 19001
+        value["steps"] = [{"type": "http", "method": "GET", "path": "/"}]
+    demo.write_text(json.dumps(value))
+    result = verify_side_task(Side(repo), demo, "path")
+    assert result.outcome == "could_not_run", result
+    assert result.phase == "start" and result.exit_code == 9
+    assert "start failed" in result.stderr and "hidden" not in result.stderr and "https://" not in result.stderr
+
+
+@pytest.mark.parametrize("secret", ['token="hidden"', '"password": "hidden"', "Bearer hidden", "ghp_hidden", "npm_hidden"])
+def test_command_failure_sanitizes_tokens_and_caps_tail(secret):
+    failure = task_storage.CommandFailure("build", 1, "old\n" * 50 + "x" * 2000 + "\n" + secret)
+    assert "hidden" not in failure.stderr and "old" not in failure.stderr
+    assert len(failure.stderr) <= 1500