search-with-proof: split compressed verifier archives, isolated pip installs, verifier module paths

Archives are gzipped and uploaded in parts of at most 90 MB with per-part
digests (edge request limit); pip installs ignore the task venv so transitive
dependencies land in products; the verifier sets NODE_PATH and the Java
classpath from products when starting the app.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/test_ax_batch.py

@@ -217,7 +217,13 @@ def test_search_executor_task_links_and_cleanup(monkeypatch, failure):
         assert session.client.command_url.endswith("?put")
         doc = calls[0][1]["manifest"]
         assert doc["spec"]["command"][2] == "openultrasast.search.executor"
-        assert {r["name"] for r in doc["spec"]["env"]} == {"COMMAND_GET_URL", "EXECUTOR_RESULT_PUT_URL", "REPO_URL", "PREPARED_PUT_URL"}
+        assert {r["name"] for r in doc["spec"]["env"]} == {
+            "COMMAND_GET_URL",
+            "EXECUTOR_RESULT_PUT_URL",
+            "REPO_URL",
+            "PREPARED_PUT_URL",
+            "PREPARED_PART_URLS",
+        }
         assert "API_KEY" not in json.dumps(doc)
         if failure:
 
@@ -227,7 +233,9 @@ def test_search_executor_task_links_and_cleanup(monkeypatch, failure):
             monkeypatch.setattr(session.client, "submit", submit)
             session.client.submit("list_files", {})
     assert calls[-1][0][:2] == ("delete", "task")
-    assert len(lane.store.deleted) == 4
+    from openultrasast.search.verify_task import MAX_PARTS
+
+    assert len(lane.store.deleted) == 4 + MAX_PARTS
 
 
 def test_capacity_backpressure_retries_same_manifest_until_deadline(tmp_path, monkeypatch):
@@ -419,3 +427,88 @@ def test_search_preparation_failure_retains_command_record():
         task.prepare({})
     assert caught.value.phase == "build" and caught.value.exit_code == 8
     assert "dependency missing" in caught.value.stderr and "hidden" not in caught.value.stderr
+
+
+def test_verifier_lane_uploads_parts_and_task_reassembles(tmp_path, monkeypatch):
+    import io
+    import subprocess
+    from types import SimpleNamespace
+
+    from benchmarks.ax.batch import AXLane
+    from openultrasast.search import executor, verify_task
+
+    monkeypatch.setattr(verify_task, "MAX_PART_BYTES", 100)
+    monkeypatch.setenv("S3_ENDPOINT", "https://files.example")
+    source = tmp_path / "source"
+    source.mkdir()
+    (source / "input").write_bytes(bytes(range(256)) * 4)
+    blob = verify_task.pack_checkout(source)
+    objects, puts = {}, []
+
+    class Store:
+        def _put(self, key, data):
+            objects["https://files.example/" + key] = data
+
+        def _get(self, key):
+            return b"completed", None
+
+        def _delete(self, key):
+            objects.pop("https://files.example/" + key, None)
+
+        def presign_get(self, key, expiry):
+            return "https://files.example/" + key
+
+        presign_put = presign_get
+
+    def put(self, url, block, timeout):
+        assert len(block) <= 100
+        puts.append(url)
+        objects[url] = block
+
+    monkeypatch.setattr(executor.URLTransport, "put", put)
+    work = Workload(IMAGE, frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}), 10, lambda *args: None, kind="search-verify")
+    lane = AXLane(SimpleNamespace(ax_bin="fake", kubeconfig=None, atespace="default"), work, Store())
+
+    def ax(*args, **kwargs):
+        if args[0] == "apply":
+            env = {value["name"]: value["value"] for value in kwargs["manifest"]["spec"]["env"]}
+            assert set(env) == {"VERIFY_INPUT_URL", "RESULT_URL"}
+            restored = io.BytesIO()
+            verify_task.assemble_parts(objects[env["VERIFY_INPUT_URL"]], restored, lambda part: objects[part["url"]])
+            assert restored.getvalue() == blob
+        return subprocess.CompletedProcess([], 0, "", "")
+
+    monkeypatch.setattr(lane, "ax", ax)
+    request = Item("verify", {"VERIFY_INPUT_URL": blob}, "result.json", ("python3",), {}, "digest")
+    output, _ = lane._execute_attempt(request, tmp_path, 0, [])
+    assert output.returncode == 0, output
+    assert len(puts) > 1
+    assert not objects  # fragment cleanup is part of the task lifecycle
+
+
+@pytest.mark.parametrize("corrupt", [False, True])
+def test_executor_prepare_checks_part_digest(monkeypatch, corrupt):
+    import hashlib
+    import io
+    import time
+    from types import SimpleNamespace
+
+    from benchmarks.ax.batch import SearchExecutorTask
+    from openultrasast.search import verify_task
+
+    monkeypatch.setattr(verify_task, "MAX_PART_BYTES", 100)
+    blob = b"compressed bytes" * 30
+    objects = {}
+    lane = SimpleNamespace(store=SimpleNamespace(_get=lambda key: (objects[key], None)))
+    task = SearchExecutorTask(lane, b"checkout")
+    metadata = verify_task.upload_parts(io.BytesIO(blob), objects.__setitem__, task.keys[4:])
+    objects[task.keys[3]] = metadata
+    if corrupt:
+        objects[task.keys[4]] = b"corrupt"
+    task.end = time.monotonic() + 60
+    task.client = SimpleNamespace(submit=lambda *args, **kwargs: {"status": "ok", "sha256": hashlib.sha256(metadata).hexdigest()})
+    if corrupt:
+        with pytest.raises(ValueError, match="part digest mismatch"):
+            task.prepare({})
+    else:
+        assert task.prepare({}) == blob

tests/test_search_executor_smoke.py

@@ -77,7 +77,13 @@ def setup(tmp_path, monkeypatch):
         if args[0] == "apply":
             doc = kwargs["manifest"]
             assert doc["metadata"]["name"].startswith("ousast-engine-search-exec-")
-            assert {x["name"] for x in doc["spec"]["env"]} == {"REPO_URL", "COMMAND_GET_URL", "EXECUTOR_RESULT_PUT_URL", "PREPARED_PUT_URL"}
+            assert {x["name"] for x in doc["spec"]["env"]} == {
+                "REPO_URL",
+                "COMMAND_GET_URL",
+                "EXECUTOR_RESULT_PUT_URL",
+                "PREPARED_PUT_URL",
+                "PREPARED_PART_URLS",
+            }
 
     lane.ax = ax
     return executor, lane, events

tests/test_search_pilot_run.py

@@ -536,7 +536,9 @@ def test_pilot_executor_deletes_task_when_submit_times_out(tmp_path, monkeypatch
     with pytest.raises(TimeoutError, match="upload deadline"), pilot.executor("a" * 40, deadline=60) as session:
         session.client.submit("list_files", {})
     assert calls[-1][0][:2] == ("delete", "task")
-    assert len(deleted) == 4
+    from openultrasast.search.verify_task import MAX_PARTS
+
+    assert len(deleted) == 4 + MAX_PARTS
     assert pilot.record["executor_task_metrics"][-1]["end"] == "execution_failure"
 
 
@@ -593,3 +595,15 @@ def test_ax_pilot_uses_verification_task_limits_for_deadline(tmp_path, monkeypat
     assert captured == [17]
     assert deadlines == [17 * (len(schema["steps"]) + 1) + 65]
     assert pilot.lane.workload is None
+
+
+def test_verify_failure_reason_is_last_300_stderr_characters(tmp_path):
+    from openultrasast.search.verify import SideRecord
+
+    pilot = make_pilot(tmp_path)
+    stderr = "Traceback:\n" + "middle of traceback " * 200 + "\nModuleNotFoundError: No module named 'sniffio'"
+    side = SideRecord("could_not_run", (), 0, "app readiness failed", phase="start", exit_code=1, stderr=stderr)
+    pilot.failure("verify/could_not_run", side.reason, side)
+    failure = pilot.record["failures"][-1]
+    assert failure["reason"] == stderr[-300:]
+    assert failure["reason"].endswith("ModuleNotFoundError: No module named 'sniffio'")

tests/test_search_task_storage.py

@@ -335,13 +335,23 @@ def test_verifier_app_virtual_memory(workspace):
 @pytest.mark.parametrize(
     "recipe,manifest,extra,expected",
     [
-        ("pip", "pyproject.toml", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "PROJECT"]),
-        ("pip", "requirements.txt", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "MANIFEST"]),
+        (
+            "pip",
+            "pyproject.toml",
+            {},
+            ["-I", "-m", "pip", "install", "--ignore-installed", "--no-warn-script-location", "--target", "PACKAGES", "PROJECT"],
+        ),
+        (
+            "pip",
+            "requirements.txt",
+            {},
+            ["-I", "-m", "pip", "install", "--ignore-installed", "--no-warn-script-location", "--target", "PACKAGES", "-r", "MANIFEST"],
+        ),
         (
             "pip",
             "pyproject.toml",
             {"requirements.txt": ""},
-            ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "REQUIREMENTS"],
+            ["-I", "-m", "pip", "install", "--ignore-installed", "--no-warn-script-location", "--target", "PACKAGES", "-r", "REQUIREMENTS"],
         ),
         ("npm", "package.json", {}, ["install", "--ignore-scripts"]),
         ("npm", "package.json", {"package-lock.json": "{}"}, ["ci", "--ignore-scripts"]),
@@ -481,27 +491,30 @@ def test_large_export_spools_to_workspace_and_excludes_caches(workspace, monkeyp
         path = repo / "nested" / directory
         path.mkdir(parents=True)
         (path / "excluded").write_bytes(b"cache")
-    seen = {}
+    uploaded = {}
 
     def upload(self, url, data, timeout):
-        assert not isinstance(data, bytes)
-        assert str(workspace) in os.readlink(f"/proc/self/fd/{data.fileno()}")
-        seen["size"] = os.fstat(data.fileno()).st_size
-        seen["sha256"] = hashlib.file_digest(data, "sha256").hexdigest()
-        data.seek(0)
-        with tarfile.open(fileobj=data, mode="r:gz") as archive:
-            names = archive.getnames()
-            assert "checkout/large.bin" in names
-            assert not any(name.endswith("excluded") for name in names)
-        data.seek(0)
+        assert isinstance(data, bytes)
+        assert len(data) <= verify_task.MAX_PART_BYTES
+        uploaded[url] = data
 
     monkeypatch.setattr(executor.URLTransport, "put", upload)
     app = executor.InProcessExecutor(repo, task_boundary=True)
     app.prepared_put_url = "https://files.example/output"
+    app.prepared_part_urls = [f"https://files.example/part-{i}" for i in range(verify_task.MAX_PARTS)]
     result = app.submit("prepare_verification", spec(), timeout_seconds=60)
     assert result["status"] == "ok", result
-    assert 128 * 1024**2 < result["archive_bytes"] == seen["size"]
-    assert result["sha256"] == seen["sha256"]
+    manifest = uploaded[app.prepared_put_url]
+    assert result["sha256"] == hashlib.sha256(manifest).hexdigest()
+    assert len(uploaded) == 3  # two bounded parts and the manifest
+    assert json.loads(manifest)["size"] == result["archive_bytes"] > 128 * 1024**2
+    with verify_task.tempfile.TemporaryFile(dir=workspace) as archive:
+        assert str(workspace) in os.readlink(f"/proc/self/fd/{archive.fileno()}")
+        verify_task.assemble_parts(manifest, archive, lambda part: uploaded[app.prepared_part_urls[part["index"]]])
+        with tarfile.open(fileobj=archive, mode="r:gz") as contents:
+            names = contents.getnames()
+            assert "checkout/large.bin" in names
+            assert not any(name.endswith("excluded") for name in names)
     assert verify_task.MAX_ARCHIVE_BYTES == 1024**3
 
 
@@ -525,3 +538,61 @@ def test_export_over_cap_reports_size_and_five_relative_directories(workspace, m
     for i in range(2, 6):
         assert f"checkout/dir{i} (" in result["reason"]
     assert "dir0" not in result["reason"] and "dir1" not in result["reason"]
+
+
+@pytest.mark.parametrize("location", ["products", "checkout"])
+def test_node_resolves_bundled_module(workspace, location):
+    if not Path("/usr/bin/node").exists():
+        pytest.skip("node unavailable")
+    repo, products = workspace / "checkout", workspace / "products"
+    (repo / "bin").mkdir(parents=True)
+    products.mkdir()
+    module = (products if location == "products" else repo) / "node_modules/bash-color"
+    module.mkdir(parents=True)
+    (module / "index.js").write_text("module.exports = 'bundled dependency';")
+    if location == "checkout":
+        destination = workspace / "bundle-check"
+        verify_task.extract_checkout(verify_task.pack_checkout(repo), destination)
+        assert (destination / "node_modules/bash-color/index.js").read_text() == (module / "index.js").read_text()
+    (repo / "bin/glance.js").write_text("if(require('bash-color') !== 'bundled dependency') process.exit(7); console.log('ready');")
+    value = spec()["demo"]
+    value["start"].update(runtime="node", path="bin/glance.js")
+    demo = workspace / "demo.json"
+    demo.write_text(json.dumps(value))
+    result = verify_side_task(Side(repo, products=products), demo, "path")
+    assert result.outcome == "observed", result
+    assert result.exit_code is None or result.exit_code == 0
+
+
+def test_java_classpath_includes_products(workspace, monkeypatch):
+    import importlib
+    import zipfile
+
+    from openultrasast.sandbox import SandboxResult
+
+    verifier = importlib.import_module("openultrasast.search.verify")
+    repo, products = workspace / "checkout", workspace / "products"
+    repo.mkdir()
+    (products / "lib").mkdir(parents=True)
+    (products / "lib/dependency.jar").write_bytes(b"dependency")
+    with zipfile.ZipFile(repo / "app.jar", "w") as jar:
+        jar.writestr("META-INF/MANIFEST.MF", "Manifest-Version: 1.0\nMain-Class: example.\n Main\n\n")
+    value = spec()["demo"]
+    value["start"].update(runtime="java", path="app.jar")
+    demo = workspace / "demo.json"
+    demo.write_text(json.dumps(value))
+    calls = []
+
+    def run(job, **kwargs):
+        calls.append(job.command)
+        assert job.command[:2] == ("/usr/bin/java", "-cp")
+        assert "/build/lib/dependency.jar" in job.command[2]
+        assert "/build/classes" in job.command[2]
+        assert job.command[3] == "example.Main"
+        assert (kwargs["mounts"]["/build"] / "lib/dependency.jar").read_bytes() == b"dependency"
+        return SandboxResult(0, "ready", "", False)
+
+    monkeypatch.setattr(verifier, "_run_task", run)
+    result = verify_side_task(Side(repo, products=products), demo, "path")
+    assert result.outcome == "observed", result
+    assert len(calls) == 2

tests/test_search_verify.py

@@ -325,7 +325,7 @@ def test_pip_products_imported_by_runtime_and_immutable(pair, monkeypatch):
 
     def runner(job, **kwargs):
         if "pip" in job.command:
-            assert "--no-index" in job.command and "--no-deps" in job.command
+            assert "--no-index" in job.command and "--ignore-installed" in job.command and "--no-warn-script-location" in job.command
             packages = kwargs["scratch"] / "packages"
             packages.mkdir()
             (packages / "dependency.py").write_text("VALUE = 42\n")

tests/test_search_verify_transport.py

@@ -67,7 +67,8 @@ def test_extract_rejects_escape_and_links(tmp_path, name, kind):
         entry.extract_checkout(data.getvalue(), tmp_path / "repo")
 
 
-def test_entry_reads_input_and_scrubs_environment(tmp_path, monkeypatch, capsys):
+@pytest.mark.parametrize("multipart", [False, True])
+def test_entry_reads_input_and_scrubs_environment(tmp_path, monkeypatch, capsys, multipart):
     root = tmp_path / "repo"
     root.mkdir()
     (root / "app.py").write_text("print(1)")
@@ -80,9 +81,18 @@ def test_entry_reads_input_and_scrubs_environment(tmp_path, monkeypatch, capsys)
     (bundle / "spec.json").write_text(json.dumps({"demo": demo(), "family": "injection", "timeout_seconds": 2}))
     blob = entry.pack_checkout(bundle)
 
+    objects = {}
+    if multipart:
+        urls = ["https://store.example/part"]
+        manifest = json.loads(entry.upload_parts(io.BytesIO(blob), objects.__setitem__, urls))
+        manifest["parts"][0]["url"] = urls[0]
+        objects["https://store.example/input"] = json.dumps(manifest).encode()
+    else:
+        objects["https://store.example/input"] = blob
+
     def download(url, limit):
-        assert limit == 1024**3
-        return blob
+        assert len(objects[url]) <= limit
+        return objects[url]
 
     monkeypatch.setattr(entry, "download", download)
     outputs = []
@@ -265,3 +275,41 @@ def test_verifier_deadline_uses_mode_budget_and_all_steps(tmp_path, monkeypatch,
     )
     dispatch(side=Side(tmp_path), demo=schema, family="injection", timeout_seconds=timeout, fresh_task=True)
     assert workloads[0].deadline == verification_run_seconds(schema, timeout) + 60 == timeout * 9 + 65
+
+
+@pytest.mark.parametrize("corrupt", [False, True])
+def test_multipart_digest_checked_before_extraction(tmp_path, monkeypatch, corrupt):
+    import hashlib
+
+    monkeypatch.setattr(entry, "MAX_PART_BYTES", 100)
+    source = tmp_path / "source"
+    source.mkdir()
+    (source / "input").write_bytes(bytes(range(256)) * 4)
+    blob = entry.pack_checkout(source)
+    objects = {}
+    urls = [f"https://store.example/part-{index}" for index in range(entry.MAX_PARTS)]
+    manifest = entry.upload_parts(io.BytesIO(blob), objects.__setitem__, urls)
+    parts = json.loads(manifest)["parts"]
+    assert len(objects) == len(parts) > 1
+    assert all(len(value) <= 100 for value in objects.values())
+    assert all(hashlib.sha256(objects[urls[p["index"]]]).hexdigest() == p["sha256"] for p in parts)
+    if corrupt:
+        objects[urls[1]] = b"x" * len(objects[urls[1]])
+    assembled = io.BytesIO()
+    if corrupt:
+        with pytest.raises(ValueError, match="part digest mismatch"):
+            entry.assemble_parts(manifest, assembled, lambda part: objects[urls[part["index"]]])
+        assert not (tmp_path / "restored").exists()
+    else:
+        entry.assemble_parts(manifest, assembled, lambda part: objects[urls[part["index"]]])
+        entry.extract_checkout(assembled, tmp_path / "restored")
+        assert (tmp_path / "restored/input").read_bytes() == (source / "input").read_bytes()
+
+
+def test_multipart_rejects_total_cap_before_fetch(monkeypatch):
+    monkeypatch.setattr(entry, "MAX_ARCHIVE_BYTES", 10)
+    with pytest.raises(ValueError, match="size limit"):
+        entry.upload_parts(io.BytesIO(b"x" * 11), lambda *args: pytest.fail("must not upload"), ["unused"])
+    manifest = json.dumps({"format": "gzip-parts-v1", "size": 11, "parts": []}).encode()
+    with pytest.raises(ValueError, match="size limit"):
+        entry.assemble_parts(manifest, io.BytesIO(), lambda *args: pytest.fail("must not fetch"))