search-with-proof: 1 GiB spooled verifier archive, tolerant text replies, transport retries

Built projects spool to /workspace without package caches (cap 1 GiB, both sides); a plain-text model reply is fed back once; mailbox PUT/GET retry for up to 90 s with idempotent re-send by seq; executor tasks are deleted on every exit path. Co-Authored-By: Claude Fable 5.1 <[email protected]>
2026-10-16 03:49:26 +00:00 · 2026-10-06 08:18:34 +02:00
parent 30cbc1ac76
commit 72ab4c8925
11 changed files with 450 additions and 95 deletions
@@ -26,7 +26,7 @@ from collections.abc import Iterator, Mapping
 from contextlib import suppress
 from dataclasses import asdict, dataclass, field
 from pathlib import Path
-from typing import Any, Protocol
+from typing import Any, BinaryIO, Protocol
 from urllib.parse import urlsplit

 from ..sandbox import SandboxJob
@@ -392,7 +392,7 @@ class InProcessExecutor:

        from .demo import preparation_command, validate_demo
        from .verify import _tree
-        from .verify_task import pack_checkout, validate_spec
+        from .verify_task import ArchiveLimit, bundle_ignore, pack_checkout, validate_spec

        if not self.task_boundary or not self.prepared_put_url:
            raise ValueError("preparation requires an executor task output")
@@ -421,17 +421,29 @@ class InProcessExecutor:
            # regular archive members. No host/external files may be exported.
            _tree(self.repo, generated_links=True)
            _tree(products, generated_links=True)
-            shutil.copytree(self.repo, bundle / "checkout", symlinks=False)
+            shutil.copytree(self.repo, bundle / "checkout", symlinks=False, ignore=bundle_ignore)
            materialized = bundle / "materialized"
-            shutil.copytree(products, materialized, symlinks=False)
+            shutil.copytree(products, materialized, symlinks=False, ignore=bundle_ignore)
            shutil.rmtree(products)
            materialized.rename(products)
            (bundle / "spec.json").write_text(json.dumps(spec))
            task_storage.check(task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else bundle)
-            blob = pack_checkout(bundle)
-            self.phase = "result upload"
-            URLTransport().put(self.prepared_put_url, blob, self._remaining())
-            return {"status": "ok", "archive_bytes": len(blob), "sha256": hashlib.sha256(blob).hexdigest()}
+            # The task entrypoint configures tempfile under disk-backed /workspace.
+            spool_root = task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else bundle.parent
+            with tempfile.TemporaryFile(dir=spool_root) as archive:
+                self.phase = "build export"
+                try:
+                    pack_checkout(bundle, archive)
+                except ArchiveLimit as exc:
+                    return {"status": "could_not_build", "phase": self.phase, "reason": str(exc), "archive_bytes": exc.archive_bytes}
+                size = archive.tell()
+                task_storage.check(spool_root, additional_bytes=size)
+                archive.seek(0)
+                digest = hashlib.file_digest(archive, "sha256").hexdigest()
+                archive.seek(0)
+                self.phase = "result upload"
+                URLTransport().put(self.prepared_put_url, archive, self._remaining())
+                return {"status": "ok", "archive_bytes": size, "sha256": digest}

    def _tree_bytes(self) -> int:
        total = 0
@@ -449,7 +461,7 @@ class InProcessExecutor:
 class Transport(Protocol):
    def get(self, url: str, timeout: float) -> bytes | None: ...

-    def put(self, url: str, data: bytes, timeout: float) -> None: ...
+    def put(self, url: str, data: bytes | BinaryIO, timeout: float) -> None: ...


 class URLTransport:
@@ -472,15 +484,35 @@ class URLTransport:
        except urllib.error.HTTPError as exc:
            if exc.code == 404:
                return None
-            raise OSError(f"object read failed: HTTP {exc.code}") from None
+            raise

-    def put(self, url: str, data: bytes, timeout: float) -> None:
-        with self.opener.open(
-            urllib.request.Request(url, data=data, method="PUT", headers={"Content-Type": "application/json"}), timeout=timeout
-        ):
+    def put(self, url: str, data: bytes | BinaryIO, timeout: float) -> None:
+        headers = {"Content-Type": "application/json"}
+        if not isinstance(data, bytes):
+            headers["Content-Length"] = str(os.fstat(data.fileno()).st_size)
+        with self.opener.open(urllib.request.Request(url, data=data, method="PUT", headers=headers), timeout=timeout):
            pass


+RETRY_SECONDS = 90.0
+
+
+def retry_mailbox(operation: Any, *, deadline: float | None = None) -> Any:
+    """Retry transient store failures; each attempt has a bounded socket timeout."""
+    end = min(time.monotonic() + RETRY_SECONDS, deadline if deadline is not None else float("inf"))
+    delay = 0.2
+    while True:
+        try:
+            return operation(max(0.001, min(10, end - time.monotonic())))
+        except (OSError, TimeoutError) as exc:
+            if isinstance(exc, urllib.error.HTTPError) and not 500 <= exc.code < 600:
+                raise
+            if time.monotonic() >= end:
+                raise
+            time.sleep(min(delay, max(0, end - time.monotonic())))
+            delay = min(delay * 2, 5)
+
+
 class ObjectStoreExecutor:
    def __init__(self, command_put_url: str, result_get_url: str, *, transport: Transport | None = None, poll_seconds: float = 0.2) -> None:
        for url in (command_put_url, result_get_url):
@@ -492,26 +524,35 @@ class ObjectStoreExecutor:
        self.poll_seconds = poll_seconds
        self.seq = 0
        self.uncertain = False
+        self.pending: Command | None = None
        self.isolation_mode = "remote-executor"

    def submit(
        self, name: str, args: dict[str, Any], *, timeout_seconds: float = 30, limits: dict[str, Any] | None = None
    ) -> dict[str, Any]:
-        if self.uncertain:
-            raise RuntimeError("previous command outcome uncertain; discard executor task")
        command = Command(self.seq + 1, name, args, timeout_seconds, limits or {})
+        if self.pending is not None:
+            if (name, args, limits or {}) != (self.pending.name, self.pending.args, self.pending.limits):
+                raise RuntimeError("previous command outcome uncertain; retry the same command")
+            command = self.pending
        payload = command.validate()
        end = time.monotonic() + timeout_seconds
        self.uncertain = True
+        self.pending = copy.deepcopy(command)
+        retry_end = end + RETRY_SECONDS
        last_error = ""
        phase = "result upload"
        while time.monotonic() < end:
            try:
                # A lost PUT acknowledgement is retried with the exact same sequence.
                phase = "command upload"
-                self.transport.put(self.command_url, payload, max(0.001, end - time.monotonic()))
-                phase = "result read"
-                raw = self.transport.get(self.result_url, max(0.001, end - time.monotonic()))
+                before = time.monotonic()
+                try:
+                    retry_mailbox(lambda timeout: self.transport.put(self.command_url, payload, timeout), deadline=retry_end)
+                    phase = "result read"
+                    raw = retry_mailbox(lambda timeout: self.transport.get(self.result_url, timeout), deadline=retry_end)
+                finally:
+                    end = min(retry_end, end + time.monotonic() - before)
                if raw is not None:
                    if len(raw) > MAX_RESULT_BYTES:
                        raise ValueError("result too large")
@@ -522,10 +563,14 @@ class ObjectStoreExecutor:
                        raise ValueError("future result sequence")
                    if result["seq"] == command.seq:
                        self.seq, self.uncertain = command.seq, False
+                        self.pending = None
                        self.isolation_mode = result.get("isolation_mode", self.isolation_mode)
                        return result
            except (OSError, TimeoutError) as exc:
+                if isinstance(exc, urllib.error.HTTPError) and not 500 <= exc.code < 600:
+                    raise
                last_error = task_storage.exception_reason(exc)
+                break
            time.sleep(min(self.poll_seconds, max(0, end - time.monotonic())))
        raise TimeoutError(f"executor {phase} deadline; {last_error or 'no result received'}")

@@ -543,7 +588,7 @@ def serve(
    end = time.monotonic() + deadline_seconds
    while time.monotonic() < end:
        try:
-            raw = transport.get(command_get_url, min(10, max(0.001, end - time.monotonic())))
+            raw = retry_mailbox(lambda timeout: transport.get(command_get_url, timeout), deadline=end)
            if raw is not None:
                if len(raw) > MAX_COMMAND_BYTES:
                    raise ValueError("command too large")
@@ -551,13 +596,16 @@ def serve(
                if command.seq != executor.seq and command.timeout_seconds > end - time.monotonic():
                    raise TimeoutError("command exceeds task lifetime")
                result = executor.execute(command)
-                transport.put(result_put_url, json.dumps(result).encode(), min(10, max(0.001, end - time.monotonic())))
+                retry_mailbox(
+                    lambda timeout, result=result: transport.put(result_put_url, json.dumps(result).encode(), timeout), deadline=end
+                )
                if result.get("stopped"):
                    return
        except (OSError, TimeoutError) as exc:
            if executor.last_result:
                executor.last_result["result_upload_reason"] = task_storage.exception_reason(exc)
                executor.last_result["result_upload_phase"] = "result upload"
+            raise

        time.sleep(min(poll_seconds, max(0, end - time.monotonic())))
    raise TimeoutError("executor task deadline")
@@ -113,7 +113,9 @@ def _tree(root: Path, *, generated_links: bool = False) -> str:
        digest.update(str(path.relative_to(root)).encode())
        digest.update(str(mode).encode())
        if path.is_file():
-            digest.update(path.read_bytes())
+            with path.open("rb") as stream:
+                while block := stream.read(65536):
+                    digest.update(block)
    if generated_links:
        visiting: set[Path] = set()
        visited: set[Path] = set()
@@ -16,7 +16,7 @@ import urllib.error
 import urllib.request
 from dataclasses import asdict
 from pathlib import Path, PurePosixPath
-from typing import Any, cast
+from typing import Any, BinaryIO, cast
 from urllib.parse import urlsplit

 from . import task_storage
@@ -47,78 +47,109 @@ def public_endpoint(url: str) -> str:
    return host


-MAX_ARCHIVE_BYTES = 64 * 1024 * 1024
-MAX_CHECKOUT_BYTES = 256 * 1024 * 1024
+MAX_ARCHIVE_BYTES = 1024**3
+MAX_CHECKOUT_BYTES = 2 * 1024**3
 MAX_FILES = 20000
 MAX_SPEC_BYTES = 70000
 MAX_RESULT_BYTES = 65536


-def pack_checkout(root: Path) -> bytes:
+CACHE_NAMES = {".npm", ".pip", ".m2", ".composer", ".gradle", "__pycache__", ".git"}
+
+
+def bundle_ignore(directory: str, names: list[str]) -> set[str]:
+    return {name for name in names if name in CACHE_NAMES or (Path(directory).name == "node_modules" and name == ".cache")}
+
+
+class ArchiveLimit(ValueError):
+    def __init__(self, size: int, directories: dict[str, int]) -> None:
+        self.archive_bytes = size
+        largest = sorted(directories.items(), key=lambda item: (-item[1], item[0]))[:5]
+        super().__init__(
+            f"archive exceeds size limit: {size} bytes > {MAX_ARCHIVE_BYTES} bytes; top 5 directories: "
+            + ", ".join(f"{name} ({count} bytes)" for name, count in largest)
+        )
+
+
+def pack_checkout(root: Path, output: BinaryIO | None = None) -> bytes:
    root = Path(root)
    if root.is_symlink() or not root.is_dir():
        raise ValueError("checkout must be a real directory")
-    buffer, total = io.BytesIO(), 0
+    buffer = output if output is not None else io.BytesIO()
+    total, count = 0, 0
+    directories: dict[str, int] = {}
    with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
-        paths = (p for p in root.rglob("*") if ".git" not in p.relative_to(root).parts)
-        for index, path in enumerate(sorted(paths)):
-            mode = path.lstat().st_mode
-            if index >= MAX_FILES or not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
-                raise ValueError("checkout contains too many files or special files")
-            total += path.stat().st_size if path.is_file() else 0
-            if total > MAX_CHECKOUT_BYTES:
-                raise ValueError("checkout exceeds size limit")
-            archive.add(path, arcname=str(path.relative_to(root)), recursive=False)
-    data = buffer.getvalue()
-    if len(data) > MAX_ARCHIVE_BYTES:
-        raise ValueError("archive exceeds size limit")
-    return data
+        for parent, dirs, files in os.walk(root, followlinks=False):
+            ignored = bundle_ignore(parent, dirs + files)
+            dirs[:] = sorted(set(dirs) - ignored)
+            for name in sorted(dirs + [name for name in files if name not in ignored]):
+                path = Path(parent) / name
+                mode = path.lstat().st_mode
+                count += 1
+                if count > MAX_FILES or not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
+                    raise ValueError("checkout contains too many files or special files")
+                size = path.stat().st_size if path.is_file() else 0
+                total += size
+                if total > MAX_CHECKOUT_BYTES:
+                    raise ValueError("checkout exceeds size limit")
+                for directory in path.relative_to(root).parents:
+                    if str(directory) != ".":
+                        directories[str(directory)] = directories.get(str(directory), 0) + size
+                archive.add(path, arcname=str(path.relative_to(root)), recursive=False)
+    if buffer.tell() > MAX_ARCHIVE_BYTES:
+        raise ArchiveLimit(buffer.tell(), directories)
+    return cast(io.BytesIO, buffer).getvalue() if output is None else b""


 def extract_checkout(data: bytes, destination: Path) -> None:
    if len(data) > MAX_ARCHIVE_BYTES:
        raise ValueError("archive exceeds size limit")
-    # Bound expansion before tarfile parses PAX headers or allocates member data.
-    with gzip.GzipFile(fileobj=io.BytesIO(data)) as compressed:
-        expanded = compressed.read(MAX_CHECKOUT_BYTES + MAX_FILES * 4096 + 1)
-    if len(expanded) > MAX_CHECKOUT_BYTES + MAX_FILES * 4096:
-        raise ValueError("expanded checkout exceeds size limit")
-    destination.mkdir(parents=True)
-    total, seen = 0, set()
-    with tarfile.open(fileobj=io.BytesIO(expanded), mode="r:") as archive:
-        for index, member in enumerate(archive):
-            path = PurePosixPath(member.name)
-            if (
-                index >= MAX_FILES
-                or path.is_absolute()
-                or ".." in path.parts
-                or not path.parts
-                or "\\" in member.name
-                or member.name in seen
-                or not (member.isfile() or member.isdir())
-            ):
-                raise ValueError("unsafe checkout archive member")
-            seen.add(member.name)
-            total += member.size
-            if member.size < 0 or total > MAX_CHECKOUT_BYTES:
-                raise ValueError("expanded checkout exceeds size limit")
-            target = destination.joinpath(*path.parts)
-            if member.isdir():
-                target.mkdir(parents=True, exist_ok=True)
-            else:
-                target.parent.mkdir(parents=True, exist_ok=True)
-                source = archive.extractfile(member)
-                if source is None:
-                    raise ValueError("missing regular checkout member")
-                with source, target.open("xb") as output:
-                    remaining = member.size
-                    while remaining:
-                        block = source.read(min(65536, remaining))
-                        if not block:
-                            raise ValueError("truncated checkout member")
-                        output.write(block)
-                        remaining -= len(block)
-                target.chmod(0o755 if member.mode & 0o111 else 0o644)
+    # Bound expansion before tarfile parses PAX headers, using disk rather than
+    # allocating the entire expanded tree in worker RAM.
+    with tempfile.TemporaryFile() as expanded:
+        with gzip.GzipFile(fileobj=io.BytesIO(data)) as compressed:
+            maximum = MAX_CHECKOUT_BYTES + MAX_FILES * 4096
+            while block := compressed.read(min(65536, maximum - expanded.tell() + 1)):
+                expanded.write(block)
+                if expanded.tell() > maximum:
+                    raise ValueError("expanded checkout exceeds size limit")
+        expanded.seek(0)
+        destination.mkdir(parents=True)
+        total, seen = 0, set()
+        with tarfile.open(fileobj=expanded, mode="r:") as archive:
+            for index, member in enumerate(archive):
+                path = PurePosixPath(member.name)
+                if (
+                    index >= MAX_FILES
+                    or path.is_absolute()
+                    or ".." in path.parts
+                    or not path.parts
+                    or "\\" in member.name
+                    or member.name in seen
+                    or not (member.isfile() or member.isdir())
+                ):
+                    raise ValueError("unsafe checkout archive member")
+                seen.add(member.name)
+                total += member.size
+                if member.size < 0 or total > MAX_CHECKOUT_BYTES:
+                    raise ValueError("expanded checkout exceeds size limit")
+                target = destination.joinpath(*path.parts)
+                if member.isdir():
+                    target.mkdir(parents=True, exist_ok=True)
+                else:
+                    target.parent.mkdir(parents=True, exist_ok=True)
+                    source = archive.extractfile(member)
+                    if source is None:
+                        raise ValueError("missing regular checkout member")
+                    with source, target.open("xb") as output:
+                        remaining = member.size
+                        while remaining:
+                            block = source.read(min(65536, remaining))
+                            if not block:
+                                raise ValueError("truncated checkout member")
+                            output.write(block)
+                            remaining -= len(block)
+                    target.chmod(0o755 if member.mode & 0o111 else 0o644)
    if not seen:
        raise ValueError("empty checkout")
    print(f"checkout_bytes={total} checkout_entries={len(seen)}", flush=True)
@@ -355,6 +355,7 @@ class Worker:
        self.evidence = set()
        self.observations = []
        malformed = 0
+        plain_text = 0
        for index in range(self.max_steps):
            messages[0]["content"] = (
                EXPLORE_PROMPT
@@ -367,7 +368,11 @@ class Worker:
            messages.append(message)
            calls = message.get("tool_calls", [])
            if not isinstance(calls, list) or not calls:
-                raise ValueError("explore must use tools or finish")
+                plain_text += 1
+                if plain_text >= 2:
+                    return {"status": "execution_failure", "failure": "second plain-text reply: " + str(message.get("content", ""))[:300]}
+                messages.append({"role": "user", "content": "Use a tool or call finish; plain text is not accepted."})
+                continue
            finishes = [call for call in calls if isinstance(call, dict) and call.get("function", {}).get("name") == "finish"]
            if finishes:
                if len(calls) > 1:
@@ -410,7 +415,7 @@ class Worker:
                            raise ValueError("incomplete demonstration")
                        schema = load_demo(self.demo)
                        family = (yaml.safe_load(task.snapshot) or {}).get("family")
-                        if family not in (None, "unknown"):
+                        if family is not None and family != "unknown":
                            validate_oracle(family, schema["oracle"])
                        fact["oracle"] = schema["oracle"]
                    return {"fact": fact}