search-with-proof: ecosystem-aware build recipes and stderr in build failures

The agent passes a manifest file or project directory; pip uses -r for
requirements files and the directory for pyproject/setup, npm ci/install runs in
the directory, composer/maven/gradle likewise. Build and verify failures carry
phase, exit code and a sanitised stderr tail.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/ax/batch.py

@@ -724,7 +724,11 @@ class SearchExecutorTask:
         remaining = min(300, self.end - time.monotonic())
         result = self.client.submit("prepare_verification", spec, timeout_seconds=remaining)
         if result.get("status") != "ok":
-            raise ValueError(result.get("reason", "could_not_build"))
+            from openultrasast.search.task_storage import CommandFailure
+
+            raise CommandFailure(
+                result.get("phase", "build"), result.get("exit_code"), result.get("stderr", ""), result.get("reason", "could_not_build")
+            )
         stored = bounded_read(lambda: self.lane.store._get(self.keys[3]), self.end - time.monotonic())
         if stored is None or hashlib.sha256(stored[0]).hexdigest() != result.get("sha256"):
             raise ValueError("executor archive missing or digest mismatch")

benchmarks/ax/search_verify.py

@@ -110,4 +110,7 @@ class AXSideDispatcher:
             max(r.scratch_peak_bytes for r in records),
             isolation_mode="task-boundary",
             chromium_no_sandbox=any(r.chromium_no_sandbox for r in records),
+            phase=last.phase,
+            exit_code=last.exit_code,
+            stderr=last.stderr,
         )

benchmarks/search/pilot_run.py

@@ -123,7 +123,7 @@ class Pilot:
         self.sides = None
         self.verifications = []
 
-    def failure(self, phase, reason):
+    def failure(self, phase, reason, command=None):
         private = tuple(str(v) for k, v in self.row.items() if k in {"repo", "file", "function", "vulnerable", "fixed"})
         private += (str(self.root), self.row["repo"].removeprefix("https://github.com/"))
         if isinstance(reason, BaseException):
@@ -135,7 +135,15 @@ class Pilot:
             safe = kind + ": " + diagnostic(message, maximum=max(0, 298 - len(kind)), private=private)
         else:
             safe = diagnostic(reason, private=private)
-        self.record["failures"].append(dict(phase=phase, reason=safe))
+        entry = dict(phase=phase, reason=safe)
+        command = command if command is not None else reason
+        if hasattr(command, "exit_code"):
+            entry.update(
+                command_phase=diagnostic(command.phase),
+                exit_code=command.exit_code,
+                stderr=diagnostic(command.stderr, maximum=1500, private=private),
+            )
+        self.record["failures"].append(entry)
 
     @contextmanager
     def executor(self, revision, deadline=900):
@@ -300,7 +308,7 @@ class Pilot:
 
                 for side in result.sides:
                     if side.outcome != "observed":
-                        self.failure("verify/" + side.outcome, "side_verification_failed")
+                        self.failure("verify/" + side.outcome, side.reason, command=side)
                 if result.outcome != "demonstrated":
                     self.failure("verify", "no_consistent_differential" if result.outcome == "inconclusive" else result.outcome)
                 end = result.outcome