search-with-proof: name the exhausted limit, keep partial evidence, tighter reservation

budget_spent carries end_detail (spend, task_spend, reason_rounds, ...); a task
whose allowance cannot cover the next call finishes with its facts; per-task
metrics in pilot records; input reservation bytes/2 + 1024 per message (measured
3-4x over-reservation before).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

src/openultrasast/search/executor.py

@@ -274,9 +274,31 @@ class InProcessExecutor:
         }
 
     def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]:
+        process_limit = None
+        try:
+            uid = os.getuid()
+            current = 0
+            for entry in Path("/proc").iterdir():
+                if not entry.name.isdecimal():
+                    continue
+                try:
+                    status = (entry / "status").read_text()
+                except FileNotFoundError:
+                    continue
+                fields = dict(line.split(":", 1) for line in status.splitlines() if ":" in line)
+                # The kernel's NPROC counter counts threads, not processes: a single IDE or agent
+                # process with 60 threads uses 60 of the allowance.
+                if int(fields["Uid"].split()[0]) == uid:
+                    current += int(fields.get("Threads", "1").strip() or 1)
+            process_limit = current + 256
+        except (OSError, ValueError, IndexError, StopIteration):
+            pass
+
         def constrain() -> None:
             resource.setrlimit(resource.RLIMIT_FSIZE, (limits.get("disk_bytes", task_storage.DEFAULT_SCRATCH_BYTES),) * 2)
-            resource.setrlimit(resource.RLIMIT_NPROC, (128, 128))
+            # NPROC counts all processes of the real uid, so an absolute 128 blocks builds on busy hosts.
+            if process_limit is not None:
+                resource.setrlimit(resource.RLIMIT_NPROC, (process_limit,) * 2)
             resource.setrlimit(resource.RLIMIT_AS, (limits.get("memory_bytes", 256 * 1024**2),) * 2)
             resource.setrlimit(resource.RLIMIT_CPU, (math.ceil(timeout) + 1,) * 2)
 
@@ -292,7 +314,7 @@ class InProcessExecutor:
             self.phase = "run"
             timed_out = False
             disk_exceeded = False
-            failure = None
+            failure: Exception | None = None
             # Monitor the full disk-backed workspace, including package caches.
             # An unnamespaced child can still hardcode RAM-backed paths: this
             # guard is not a replacement for the operator's memory limit.
@@ -391,7 +413,10 @@ class InProcessExecutor:
                 if recipe["recipe"] == "maven":
                     argv += ["package", "-DskipTests"]
                 elif recipe["recipe"] == "gradle":
-                    argv += ["assemble"]
+                    # The project pins its Gradle version through its wrapper.
+                    # Invoke with sh because checkout archives may lose execute bits.
+                    wrapper = self._path(str(path.relative_to(self.repo) / "gradlew"))
+                    argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"]
                 built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES})
                 if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
                     return {**built, "status": "could_not_build"}