search-with-proof: split compressed verifier archives, isolated pip installs, verifier module paths

Archives are gzipped and uploaded in parts of at most 90 MB with per-part
digests (edge request limit); pip installs ignore the task venv so transitive
dependencies land in products; the verifier sets NODE_PATH and the Java
classpath from products when starting the app.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

src/openultrasast/search/demo.py

@@ -59,7 +59,19 @@ MAX_VALUE_BYTES = 16384
 # an enumerated build goal, never arbitrary package-manager flags or scripts.
 BUILD_RECIPES = {
     "none": (),
-    "pip": ("/usr/bin/python3", "-I", "-m", "pip", "install", "--no-index", "--no-deps", "--target", "/scratch/packages", "-r"),
+    "pip": (
+        "/usr/bin/python3",
+        "-I",
+        "-m",
+        "pip",
+        "install",
+        "--no-index",
+        "--ignore-installed",
+        "--no-warn-script-location",
+        "--target",
+        "/scratch/packages",
+        "-r",
+    ),
     "npm": ("/usr/bin/npm", "ci", "--offline", "--ignore-scripts", "--prefix"),
     "composer": ("/usr/bin/composer", "install", "--no-plugins", "--no-scripts", "--no-interaction", "--working-dir"),
     "maven": ("/usr/bin/mvn", "--offline", "-f"),
@@ -241,7 +253,8 @@ def preparation_command(recipe: str, path: Path, checkout: Path, products: Path)
             "-m",
             "pip",
             "install",
-            "--no-deps",
+            "--ignore-installed",
+            "--no-warn-script-location",
             "--target",
             str(products / "packages"),
             *(["-r", str(target)] if requirements else [str(target)]),

src/openultrasast/search/executor.py

@@ -107,6 +107,7 @@ class InProcessExecutor:
         self.last_result: dict[str, Any] = {}
         self.isolation_mode = "unprobed"
         self.prepared_put_url: str | None = None
+        self.prepared_part_urls: list[str] = []
 
     def submit(
         self, name: str, args: dict[str, Any], *, timeout_seconds: float = 30, limits: dict[str, Any] | None = None
@@ -392,7 +393,7 @@ class InProcessExecutor:
 
         from .demo import preparation_command, validate_demo
         from .verify import _tree
-        from .verify_task import ArchiveLimit, bundle_ignore, pack_checkout, validate_spec
+        from .verify_task import MAX_PART_BYTES, ArchiveLimit, bundle_ignore, pack_checkout, upload_parts, validate_spec
 
         if not self.task_boundary or not self.prepared_put_url:
             raise ValueError("preparation requires an executor task output")
@@ -442,7 +443,16 @@ class InProcessExecutor:
                 digest = hashlib.file_digest(archive, "sha256").hexdigest()
                 archive.seek(0)
                 self.phase = "result upload"
-                URLTransport().put(self.prepared_put_url, archive, self._remaining())
+                if self.prepared_part_urls:
+                    manifest = upload_parts(
+                        archive, lambda url, block: URLTransport().put(url, block, self._remaining()), self.prepared_part_urls
+                    )
+                    URLTransport().put(self.prepared_put_url, manifest, self._remaining())
+                    digest = hashlib.sha256(manifest).hexdigest()
+                elif size <= MAX_PART_BYTES:
+                    URLTransport().put(self.prepared_put_url, archive, self._remaining())
+                else:
+                    raise ValueError("multipart archive URLs required")
                 return {"status": "ok", "archive_bytes": size, "sha256": digest}
 
     def _tree_bytes(self) -> int:
@@ -656,6 +666,7 @@ def main() -> None:
     result_url = args.result_put_url or os.environ.get("EXECUTOR_RESULT_PUT_URL")
     repo_url = os.environ.get("REPO_URL")
     prepared_url = os.environ.get("PREPARED_PUT_URL")
+    prepared_parts = json.loads(os.environ.get("PREPARED_PART_URLS", "[]"))
     # Clearing Python's environment cannot remove initial /proc/self/environ.
     # Refuse a misconfigured task before any repository input is opened.
     inherited = sorted(
@@ -678,6 +689,7 @@ def main() -> None:
         unpack_checkout(download(repo_url, 128 * 1024**2, readiness_budget=min(60, args.deadline)), args.repo)
     executor = InProcessExecutor(args.repo, task_boundary=args.task_boundary)
     executor.prepared_put_url = prepared_url
+    executor.prepared_part_urls = prepared_parts
     serve(executor, command_url, result_url, deadline_seconds=args.deadline)
 
 

src/openultrasast/search/verify.py

@@ -18,6 +18,7 @@ import stat
 import subprocess
 import tempfile
 import time
+import zipfile
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import suppress
@@ -380,6 +381,21 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                     )
                     command = ("/usr/bin/python3", "-I", "-c", bootstrap, "/workspace/" + start_spec["path"], *start_spec["arguments"])
 
+                if start_spec["runtime"] == "java":
+                    # -jar ignores CLASSPATH. Use the jar's declared main class
+                    # with the verifier-owned product directories and jars.
+                    with zipfile.ZipFile(entry) as jar, jar.open("META-INF/MANIFEST.MF") as jar_manifest:
+                        metadata = jar_manifest.read(65537)
+                    if len(metadata) > 65536:
+                        return record("could_not_build", "oversized Java manifest")
+                    metadata = metadata.replace(b"\r\n", b"\n").replace(b"\n ", b"")
+                    main = re.search(rb"(?mi)^Main-Class: ([\w.$]+)\s*$", metadata.split(b"\n\n", 1)[0])
+                    if main is None:
+                        return record("could_not_build", "Java jar missing Main-Class")
+                    classpath = ["/workspace/" + start_spec["path"], "/build/", "/build/classes"]
+                    classpath.extend("/build/" + str(path.relative_to(products)) for path in sorted(products.rglob("*.jar")))
+                    command = ("/usr/bin/java", "-cp", ":".join(classpath), main[1].decode(), *start_spec["arguments"])
+
                 oracle = oracle_for(family, browser)
                 assert oracle is not None
                 try:
@@ -393,6 +409,8 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                         },
                         **oracle.prepare(canary),
                     }
+                    if start_spec["runtime"] == "node":
+                        app_env["NODE_PATH"] = "/build/node_modules"
                     app_scratch = canary.root if family == "command" else root / "app"
                     app_scratch.mkdir(exist_ok=True)
                     fixture = canary.root
@@ -483,7 +501,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
         return record("could_not_run", str(exc), exc.phase, exc)
     except task_storage.ScratchLimit as exc:
         return record("could_not_build", exception_reason(exc))
-    except (OSError, ValueError, AssertionError, KeyError, IndexError, TypeError) as exc:
+    except (OSError, ValueError, AssertionError, KeyError, IndexError, TypeError, zipfile.BadZipFile) as exc:
         return record("could_not_run", "verification unavailable or refused: " + exception_reason(exc))
 
 

src/openultrasast/search/verify_task.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import gzip
+import hashlib
 import io
 import ipaddress
 import json
@@ -48,6 +49,9 @@ def public_endpoint(url: str) -> str:
 
 
 MAX_ARCHIVE_BYTES = 1024**3
+MAX_PART_BYTES = 90_000_000
+MAX_PARTS = (MAX_ARCHIVE_BYTES + MAX_PART_BYTES - 1) // MAX_PART_BYTES
+MAX_MANIFEST_BYTES = 65536
 MAX_CHECKOUT_BYTES = 2 * 1024**3
 MAX_FILES = 20000
 MAX_SPEC_BYTES = 70000
@@ -78,7 +82,7 @@ def pack_checkout(root: Path, output: BinaryIO | None = None) -> bytes:
     buffer = output if output is not None else io.BytesIO()
     total, count = 0, 0
     directories: dict[str, int] = {}
-    with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
+    with tarfile.open(fileobj=buffer, mode="w:gz", compresslevel=6) as archive:
         for parent, dirs, files in os.walk(root, followlinks=False):
             ignored = bundle_ignore(parent, dirs + files)
             dirs[:] = sorted(set(dirs) - ignored)
@@ -101,13 +105,70 @@ def pack_checkout(root: Path, output: BinaryIO | None = None) -> bytes:
     return cast(io.BytesIO, buffer).getvalue() if output is None else b""
 
 
-def extract_checkout(data: bytes, destination: Path) -> None:
-    if len(data) > MAX_ARCHIVE_BYTES:
+def upload_parts(source: BinaryIO, put: Any, urls: list[str]) -> bytes:
+    """Publish bounded gzip fragments, then return the ordered digest manifest."""
+    source.seek(0, 2)
+    size = source.tell()
+    if not 0 < size <= MAX_ARCHIVE_BYTES:
+        raise ValueError("archive exceeds size limit")
+    if len(urls) < (size + MAX_PART_BYTES - 1) // MAX_PART_BYTES:
+        raise ValueError("missing archive part URLs")
+    source.seek(0)
+    parts: list[dict[str, Any]] = []
+    while block := source.read(MAX_PART_BYTES):
+        index = len(parts)
+        put(urls[index], block)
+        parts.append({"index": index, "size": len(block), "sha256": hashlib.sha256(block).hexdigest()})
+    return json.dumps({"format": "gzip-parts-v1", "size": size, "parts": parts}).encode()
+
+
+def assemble_parts(data: bytes, output: BinaryIO, get: Any) -> None:
+    """Validate the complete manifest before fetching, and each part before use."""
+    if len(data) > MAX_MANIFEST_BYTES:
+        raise ValueError("oversized archive manifest")
+    manifest = json.loads(data)
+    if not isinstance(manifest, dict) or manifest.get("format") != "gzip-parts-v1":
+        raise ValueError("invalid archive manifest")
+    parts = manifest.get("parts")
+    size = manifest.get("size")
+    if type(size) is not int or not 0 < size <= MAX_ARCHIVE_BYTES:
+        raise ValueError("archive exceeds size limit")
+    if not isinstance(parts, list) or not 1 <= len(parts) <= MAX_PARTS:
+        raise ValueError("invalid archive parts")
+    total = 0
+    for index, part in enumerate(parts):
+        if (
+            not isinstance(part, dict)
+            or type(part.get("index")) is not int
+            or part["index"] != index
+            or type(part.get("size")) is not int
+            or not 0 < part["size"] <= MAX_PART_BYTES
+            or not isinstance(part.get("sha256"), str)
+            or len(part["sha256"]) != 64
+        ):
+            raise ValueError("invalid archive part")
+        total += part["size"]
+    if total != size:
+        raise ValueError("archive part sizes mismatch")
+    for part in parts:
+        block = get(part)
+        if len(block) != part["size"] or hashlib.sha256(block).hexdigest() != part["sha256"]:
+            raise ValueError("archive part digest mismatch")
+        output.write(block)
+    output.seek(0)
+
+
+def extract_checkout(data: bytes | BinaryIO, destination: Path) -> None:
+    compressed_source = io.BytesIO(data) if isinstance(data, bytes) else data
+    compressed_source.seek(0, 2)
+    size = compressed_source.tell()
+    compressed_source.seek(0)
+    if size > MAX_ARCHIVE_BYTES:
         raise ValueError("archive exceeds size limit")
     # Bound expansion before tarfile parses PAX headers, using disk rather than
     # allocating the entire expanded tree in worker RAM.
     with tempfile.TemporaryFile() as expanded:
-        with gzip.GzipFile(fileobj=io.BytesIO(data)) as compressed:
+        with gzip.GzipFile(fileobj=compressed_source) as compressed:
             maximum = MAX_CHECKOUT_BYTES + MAX_FILES * 4096
             while block := compressed.read(min(65536, maximum - expanded.tell() + 1)):
                 expanded.write(block)
@@ -232,7 +293,12 @@ def task_main() -> None:
     with tempfile.TemporaryDirectory(prefix="ousast-verify-task-") as temporary:
         root = Path(temporary)
         try:
-            extract_checkout(archive, root / "input")
+            if archive.startswith(b"\x1f\x8b"):
+                extract_checkout(archive, root / "input")
+            else:
+                with tempfile.TemporaryFile() as compressed:
+                    assemble_parts(archive, compressed, lambda part: download(part["url"], part["size"]))
+                    extract_checkout(compressed, root / "input")
             task_storage.check(task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else root)
             bundle = root / "input"
             if {p.name for p in bundle.iterdir()} != {"checkout", "products", "spec.json"}: