search-with-proof: diagnosable failures, pip install race, no duplicate tasks

Reasons carry type plus a sanitised message tail; command failures carry phase,
exit code and bounded stderr. Scratch walk tolerates directories pip removes
mid-walk; writable /workspace/home and runtime paths. Capacity retries delete a
possibly created task before resubmitting. Docker smoke checks the image first.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/openultrasast/search/executor.py

@@ -34,7 +34,7 @@ from . import _sandbox, task_storage
 
 MAX_COMMAND_BYTES = 65536
 MAX_RESULT_BYTES = 65536
-SAFE_ENV = {"PATH": "/usr/bin:/bin", "LANG": "C.UTF-8"}
+SAFE_ENV = {"PATH": "/venv/bin:/opt/java/openjdk/bin:/usr/local/bin:/usr/bin:/bin", "LANG": "C.UTF-8", "JAVA_HOME": "/opt/java/openjdk"}
 
 
 def clean_environment(_source: Mapping[str, str] | None = None) -> dict[str, str]:
@@ -111,7 +111,7 @@ class InProcessExecutor:
     ) -> dict[str, Any]:
         result = self.execute(Command(self.seq + 1, name, args, timeout_seconds, limits or {}))
         if result.get("error") in {"ValueError", "KeyError", "TypeError", "FileNotFoundError", "OSError"}:
-            raise ValueError("repository command rejected")
+            raise ValueError(result.get("reason", "repository command rejected"))
         return result
 
     def execute(self, command: Command) -> dict[str, Any]:
@@ -126,13 +126,28 @@ class InProcessExecutor:
             raise ValueError("out of order sequence")
         started = time.monotonic()
         self.deadline = started + command.timeout_seconds
+        self.phase = "run"
         try:
             result = self._execute(command)
-        except task_storage.ScratchLimit:
+        except task_storage.ScratchLimit as exc:
             self.stopped = True
-            result = {"status": "could_not_build", "reason": "scratch limit", "stopped": True}
-        except (ValueError, OSError, KeyError, TypeError, TimeoutError) as exc:
-            result = {"error": type(exc).__name__, "timed_out": isinstance(exc, TimeoutError)}
+            result = {
+                "status": "could_not_build",
+                "reason": task_storage.exception_reason(exc, private=(str(self.repo), self.repo.name)),
+                "phase": "scratch guard",
+                "exit_code": None,
+                "stderr": "",
+                "stopped": True,
+            }
+        except (ValueError, OSError, KeyError, TypeError, TimeoutError, subprocess.SubprocessError) as exc:
+            result = {
+                "error": type(exc).__name__,
+                "reason": task_storage.exception_reason(exc, private=(str(self.repo), self.repo.name)),
+                "phase": self.phase,
+                "exit_code": None,
+                "stderr": "",
+                "timed_out": isinstance(exc, TimeoutError),
+            }
         result["wall_seconds"] = time.monotonic() - started
         result = _bounded({"seq": command.seq, **result}, min(MAX_RESULT_BYTES, command.limits.get("result_bytes", MAX_RESULT_BYTES)))
         self.seq, self.last_digest, self.last_result = command.seq, digest, copy.deepcopy(result)
@@ -179,7 +194,9 @@ class InProcessExecutor:
     def _execute(self, command: Command) -> dict[str, Any]:
         name, args = command.name, command.args
         if self.task_boundary:
+            self.phase = "scratch guard"
             task_storage.check(task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else self.repo)
+            self.phase = "run"
         if name == "prepare_verification":
             return self._prepare_verification(args)
         if name == "stop":
@@ -240,10 +257,18 @@ class InProcessExecutor:
                 wall_timeout_seconds=self._remaining(),
             )
         return {
+            **(
+                {
+                    "phase": "timeout" if result.timed_out else "run",
+                    "reason": "TimeoutError: command deadline" if result.timed_out else f"CommandFailure: exit code {result.exit_code}",
+                }
+                if result.timed_out or result.exit_code
+                else {}
+            ),
             "isolation_mode": self.isolation_mode,
             "exit_code": result.exit_code,
             "stdout": result.stdout[:16384],
-            "stderr": result.stderr[-16384:],
+            "stderr": task_storage.diagnostic(result.stderr, maximum=4000),
             "timed_out": result.timed_out,
             "truncated": len(result.stdout) > 16384 or len(result.stderr) > 16384,
         }
@@ -259,12 +284,15 @@ class InProcessExecutor:
         initial_bytes = self._tree_bytes()
         guard_root = task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else self.repo
         peak_bytes = task_storage.check(guard_root)
+        self.phase = "spawn"
         with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
             proc = subprocess.Popen(
                 argv, cwd=self.repo, env=clean_environment(), stdout=out, stderr=err, start_new_session=True, preexec_fn=constrain
             )
+            self.phase = "run"
             timed_out = False
             disk_exceeded = False
+            failure = None
             # Monitor the full disk-backed workspace, including package caches.
             # An unnamespaced child can still hardcode RAM-backed paths: this
             # guard is not a replacement for the operator's memory limit.
@@ -290,16 +318,36 @@ class InProcessExecutor:
                         break
                     except subprocess.TimeoutExpired:
                         self._remaining()
-            except TimeoutError:
+            except task_storage.ScratchLimit as exc:
+                disk_exceeded = True
+                failure = exc
+                self.stopped = True
+            except TimeoutError as exc:
                 timed_out = True
+                failure = exc
+            except OSError as exc:
+                self.phase = "scratch guard"
+                failure = exc
             finally:
                 with suppress(ProcessLookupError):
                     os.killpg(proc.pid, signal.SIGKILL)
                 proc.wait()
             out.seek(0)
             err.seek(max(0, os.fstat(err.fileno()).st_size - 16384))
+            stderr = task_storage.diagnostic(err.read().decode(errors="replace"), maximum=4000)
+            phase = "scratch guard" if disk_exceeded else "timeout" if timed_out else self.phase
+            if disk_exceeded:
+                self.stopped = True
+            failed = failure or (task_storage.ScratchLimit("scratch limit") if disk_exceeded else None)
+            reason = (
+                task_storage.exception_reason(failed)
+                if failed
+                else (f"CommandFailure: exit code {proc.returncode}" if proc.returncode else "")
+            )
             return {
-                **({"status": "could_not_build", "reason": "scratch limit"} if disk_exceeded else {}),
+                **({"status": "could_not_build", "stopped": self.stopped} if disk_exceeded else {}),
+                **({"reason": reason, "phase": phase} if reason else {}),
+                **({"error": type(failure).__name__} if failure and not (timed_out or disk_exceeded) else {}),
                 "isolation_mode": "task-boundary",
                 "exit_code": proc.returncode,
                 "timed_out": timed_out,
@@ -309,7 +357,7 @@ class InProcessExecutor:
                 "stdout_bytes": os.fstat(out.fileno()).st_size,
                 "stderr_bytes": os.fstat(err.fileno()).st_size,
                 "stdout": out.read(16384).decode(errors="replace"),
-                "stderr": err.read(16384).decode(errors="replace"),
+                "stderr": stderr,
                 "truncated": os.fstat(out.fileno()).st_size > 16384 or os.fstat(err.fileno()).st_size > 16384,
             }
 
@@ -345,10 +393,8 @@ class InProcessExecutor:
                 elif recipe["recipe"] == "gradle":
                     argv += ["assemble"]
                 built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES})
-                if built.get("reason") == "scratch limit":
-                    raise task_storage.ScratchLimit("scratch limit")
-                if built["exit_code"] or built["timed_out"]:
-                    return {"status": "could_not_build", "reason": "build recipe failed: " + built["stderr"][-2000:]}
+                if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
+                    return {**built, "status": "could_not_build"}
             # Validate internal generated links before materializing them as
             # regular archive members. No host/external files may be exported.
             _tree(self.repo, generated_links=True)
@@ -361,6 +407,7 @@ class InProcessExecutor:
             (bundle / "spec.json").write_text(json.dumps(spec))
             task_storage.check(task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else bundle)
             blob = pack_checkout(bundle)
+            self.phase = "result upload"
             URLTransport().put(self.prepared_put_url, blob, self._remaining())
             return {"status": "ok", "archive_bytes": len(blob), "sha256": hashlib.sha256(blob).hexdigest()}
 
@@ -403,7 +450,7 @@ class URLTransport:
         except urllib.error.HTTPError as exc:
             if exc.code == 404:
                 return None
-            raise OSError("object read failed") from None
+            raise OSError(f"object read failed: HTTP {exc.code}") from None
 
     def put(self, url: str, data: bytes, timeout: float) -> None:
         with self.opener.open(
@@ -434,10 +481,14 @@ class ObjectStoreExecutor:
         payload = command.validate()
         end = time.monotonic() + timeout_seconds
         self.uncertain = True
+        last_error = ""
+        phase = "result upload"
         while time.monotonic() < end:
             try:
                 # A lost PUT acknowledgement is retried with the exact same sequence.
+                phase = "command upload"
                 self.transport.put(self.command_url, payload, max(0.001, end - time.monotonic()))
+                phase = "result read"
                 raw = self.transport.get(self.result_url, max(0.001, end - time.monotonic()))
                 if raw is not None:
                     if len(raw) > MAX_RESULT_BYTES:
@@ -451,10 +502,10 @@ class ObjectStoreExecutor:
                         self.seq, self.uncertain = command.seq, False
                         self.isolation_mode = result.get("isolation_mode", self.isolation_mode)
                         return result
-            except (OSError, TimeoutError):
-                pass
+            except (OSError, TimeoutError) as exc:
+                last_error = task_storage.exception_reason(exc)
             time.sleep(min(self.poll_seconds, max(0, end - time.monotonic())))
-        raise TimeoutError("executor result deadline")
+        raise TimeoutError(f"executor {phase} deadline; {last_error or 'no result received'}")
 
 
 def serve(
@@ -481,8 +532,11 @@ def serve(
                 transport.put(result_put_url, json.dumps(result).encode(), min(10, max(0.001, end - time.monotonic())))
                 if result.get("stopped"):
                     return
-        except (OSError, TimeoutError):
-            pass
+        except (OSError, TimeoutError) as exc:
+            if executor.last_result:
+                executor.last_result["result_upload_reason"] = task_storage.exception_reason(exc)
+                executor.last_result["result_upload_phase"] = "result upload"
+
         time.sleep(min(poll_seconds, max(0, end - time.monotonic())))
     raise TimeoutError("executor task deadline")
 
@@ -534,11 +588,14 @@ def main() -> None:
     prepared_url = os.environ.get("PREPARED_PUT_URL")
     # Clearing Python's environment cannot remove initial /proc/self/environ.
     # Refuse a misconfigured task before any repository input is opened.
-    if any(
-        value and any(word in key.upper() for word in ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL"))
+    inherited = sorted(
+        key
         for key, value in os.environ.items()
-    ):
-        raise SystemExit("executor refuses inherited credentials")
+        if value and any(word in key.upper() for word in ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL"))
+    )
+    if inherited:
+        # Names only, never values, so a misconfigured task can be diagnosed from the worker log.
+        raise SystemExit("executor refuses inherited credentials: " + ", ".join(inherited))
     scratch_limit = task_storage.configure()
     os.environ.clear()
     os.environ.update(clean_environment())

src/openultrasast/search/probe.py

@@ -22,6 +22,7 @@ from typing import Any
 
 from . import _sandbox
 from .oracles import BrowserExecutor
+from .task_storage import exception_reason
 from .verify import Side, verify
 
 FAMILIES = ("sql", "path", "command", "ssrf", "xss")
@@ -128,13 +129,13 @@ def probe(root: Path) -> dict[str, Any]:
                     if side.reason.startswith("verification unavailable or refused:"):
                         raise _sandbox.IsolationUnavailable(f"{family}/{name}: {side.reason}")
     except Exception as exc:
-        record.update(status="instrument_failure", exception=f"{type(exc).__name__}: {exc}")
+        record.update(status="instrument_failure", exception=exception_reason(exc))
         namespaces = record["namespaces"]
         if namespaces and all(namespaces[name]["exit_code"] != 0 for name in ("user", "pid", "net", "ipc", "uts", "mount")):
             record["isolation_mode"] = "task-boundary"
             record["requires_fresh_side_tasks"] = True
         if record["isolation_check"] is None:
-            record["isolation_check"] = {"status": "instrument_failure", "exception": str(exc)}
+            record["isolation_check"] = {"status": "instrument_failure", "exception": exception_reason(exc)}
     finally:
         record["wall_seconds"] = time.monotonic() - start
         record["peak_rss_kib"] = {
@@ -165,8 +166,9 @@ def main(argv: list[str] | None = None) -> int:
             with urllib.request.urlopen(request, timeout=30) as response:
                 if not 200 <= response.status < 300:
                     return 1
-        except Exception:
-            # Presigned URL is a bearer secret; never print upload exceptions.
+        except Exception as exc:
+            record.update(status="instrument_failure", phase="result upload", reason=exception_reason(exc))
+            print(json.dumps(record), flush=True)
             return 1
     return 0 if record["status"] == "ok" else 1
 

src/openultrasast/search/task_storage.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import re
 import stat
 import subprocess
 import tempfile
@@ -23,6 +24,7 @@ def environment() -> dict[str, str]:
         return {}
     root = str(WORKSPACE)
     return {
+        "HOME": root + "/home",
         "TMPDIR": root + "/tmp",
         "npm_config_cache": root + "/.npm",
         "MAVEN_OPTS": "-Dmaven.repo.local=" + root + "/.m2",
@@ -39,6 +41,7 @@ def configure() -> int:
     env = environment()
     if env:
         Path(env["TMPDIR"]).mkdir(parents=True, exist_ok=True)
+        Path(env["HOME"]).mkdir(parents=True, exist_ok=True)
         os.environ.update(env)
         # tempfile may have cached /tmp before this entry point ran.
         tempfile.tempdir = env["TMPDIR"]
@@ -53,7 +56,8 @@ def check(root: Path | None = None, limit: int | None = None, *, additional_byte
         raise OSError("scratch root unavailable")
 
     def unreadable(error: OSError) -> None:
-        raise error
+        if not isinstance(error, FileNotFoundError):
+            raise error
 
     total = additional_bytes
     if total > limit:
@@ -91,3 +95,25 @@ def communicate(proc: subprocess.Popen[Any], data: bytes, timeout: float, root:
             first = False
             if time.monotonic() >= end:
                 raise
+
+
+def diagnostic(value: object, *, maximum: int = 300, private: tuple[str, ...] = ()) -> str:
+    """Sanitize before truncation so a clipped secret cannot escape redaction."""
+    text = str(value)
+    for key, secret in os.environ.items():
+        if len(secret) >= 4 and any(word in key.upper() for word in ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL")):
+            text = text.replace(secret, "[redacted]")
+    text = re.sub(r"(?i)bearer\s+[^\s,;\"']+", "Bearer [redacted]", text)
+    text = re.sub(r"(?i)(?:https?|s3)://[^\s<>\"']+", "[redacted-url]", text)
+    text = re.sub(
+        r"(?i)([\w-]*(?:authorization|signature|credential|token|secret|password|key))\s*[:=]\s*[^\s,;&\"']+", r"\1=[redacted]", text
+    )
+    for name in sorted(set(private), key=len, reverse=True):
+        if name:
+            text = text.replace(name, "[repository]")
+    return "\n".join(text.splitlines()[-20:])[-maximum:]
+
+
+def exception_reason(exc: BaseException, *, private: tuple[str, ...] = ()) -> str:
+    prefix = type(exc).__name__ + ": "
+    return prefix + diagnostic(exc, maximum=max(0, 300 - len(prefix)), private=private)

src/openultrasast/search/verify.py

@@ -29,6 +29,7 @@ from ..sandbox import SandboxJob, SandboxResult
 from . import _sandbox, task_storage
 from .demo import BUILD_RECIPES, RUNTIMES, capture_output, load_demo, substitute
 from .oracles import BrowserExecutor, Canary, Oracle, oracle_for
+from .task_storage import exception_reason
 
 
 @dataclass(frozen=True)
@@ -414,10 +415,10 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                 finally:
                     oracle.close()
         return record("observed", "fresh task run completed" if task_boundary else "three fresh runs completed")
-    except task_storage.ScratchLimit:
-        return record("could_not_build", "scratch limit")
+    except task_storage.ScratchLimit as exc:
+        return record("could_not_build", exception_reason(exc))
     except (OSError, ValueError, AssertionError, KeyError, IndexError, TypeError) as exc:
-        return record("could_not_run", f"verification unavailable or refused: {exc}")
+        return record("could_not_run", "verification unavailable or refused: " + exception_reason(exc))
 
 
 def _run_task(
@@ -508,7 +509,7 @@ def verify(
     try:
         load_demo(artefact)
     except (OSError, ValueError) as exc:
-        return VerificationRecord("inconclusive", (), time.monotonic() - started, "invalid declarative demo: " + str(exc))
+        return VerificationRecord("inconclusive", (), time.monotonic() - started, "invalid declarative demo: " + exception_reason(exc))
     mode = "task-boundary" if task_dispatcher is not None else _sandbox.isolation_mode()
     if task_dispatcher is None:
         try:

src/openultrasast/search/verify_task.py

@@ -212,8 +212,8 @@ def task_main() -> None:
             demo.mkdir()
             (demo / "demo.json").write_text(json.dumps(spec["demo"]))
             result = run_side(Side(bundle / "checkout", products=bundle / "products"), demo, spec["family"], spec["timeout_seconds"])
-        except task_storage.ScratchLimit:
-            result = SideRecord("could_not_build", (), 0, "scratch limit", isolation_mode="task-boundary")
+        except task_storage.ScratchLimit as exc:
+            result = SideRecord("could_not_build", (), 0, task_storage.exception_reason(exc), isolation_mode="task-boundary")
         upload(urls["RESULT_URL"], json.dumps(asdict(result)).encode())
 
 

src/openultrasast/search/worker.py

@@ -20,6 +20,7 @@ from . import _sandbox
 from .coordinator import SearchTask
 from .demo import DEMO_SCHEMA, load_demo, validate_demo
 from .executor import ObjectStoreExecutor
+from .task_storage import exception_reason
 
 
 def _schema(name: str, description: str, properties: dict[str, Any], required: list[str]) -> dict[str, Any]:
@@ -286,7 +287,7 @@ class Worker:
                     self.evidence.add(ref)
                     result["evidence_ref"] = ref
                 except (ValueError, OSError, KeyError, TypeError) as exc:
-                    result = {"error": type(exc).__name__}
+                    result = {"error": type(exc).__name__, "reason": exception_reason(exc)}
                 messages.append({"role": "tool", "tool_call_id": call["id"], "content": json.dumps(result)})
         return {"status": "execution_failure", "failure": "tool step limit exceeded"}
 
@@ -307,18 +308,18 @@ class Worker:
                 "cost_usd": meter.spent,
                 "isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"),
             }
-        except _sandbox.IsolationUnavailable:
+        except _sandbox.IsolationUnavailable as exc:
             return {
                 "status": "instrument_failure",
-                "failure": "sandbox isolation unavailable",
+                "failure": exception_reason(exc),
                 "cost_usd": meter.spent,
                 "isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"),
             }
         except Exception as exc:
-            # Provider exceptions can contain keys/URLs. Never persist their text.
+            # Preserve the diagnostic after removing credentials and URLs.
             return {
                 "status": "execution_failure",
-                "failure": type(exc).__name__,
+                "failure": exception_reason(exc),
                 "cost_usd": meter.spent,
                 "isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"),
             }