search-with-proof: 1 GiB spooled verifier archive, tolerant text replies, transport retries

Built projects spool to /workspace without package caches (cap 1 GiB, both
sides); a plain-text model reply is fed back once; mailbox PUT/GET retry for up
to 90 s with idempotent re-send by seq; executor tasks are deleted on every exit
path.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/test_ax_batch.py

@@ -177,7 +177,8 @@ def test_search_environment(step):
             manifest("ousast-engine-search-1", workload(), it, urls)
 
 
-def test_search_executor_task_links_and_cleanup():
+@pytest.mark.parametrize("failure", [None, TimeoutError, KeyboardInterrupt])
+def test_search_executor_task_links_and_cleanup(monkeypatch, failure):
     from types import SimpleNamespace
 
     from benchmarks.ax.batch import SearchExecutorTask
@@ -209,13 +210,22 @@ def test_search_executor_task_links_and_cleanup():
         clock=__import__("time").monotonic,
         pause=lambda _: None,
     )
-    with SearchExecutorTask(lane, b"archive", deadline=60) as session:
+    from contextlib import nullcontext
+
+    with pytest.raises(failure) if failure else nullcontext(), SearchExecutorTask(lane, b"archive", deadline=60) as session:
         assert session.name.startswith("ousast-engine-search-exec-")
         assert session.client.command_url.endswith("?put")
         doc = calls[0][1]["manifest"]
         assert doc["spec"]["command"][2] == "openultrasast.search.executor"
         assert {r["name"] for r in doc["spec"]["env"]} == {"COMMAND_GET_URL", "EXECUTOR_RESULT_PUT_URL", "REPO_URL", "PREPARED_PUT_URL"}
         assert "API_KEY" not in json.dumps(doc)
+        if failure:
+
+            def submit(*args, **kwargs):
+                raise failure("transport interrupted")
+
+            monkeypatch.setattr(session.client, "submit", submit)
+            session.client.submit("list_files", {})
     assert calls[-1][0][:2] == ("delete", "task")
     assert len(lane.store.deleted) == 4
 

tests/test_search_executor.py

@@ -77,7 +77,7 @@ def test_mailbox_timeout():
     with pytest.raises(TimeoutError):
         client.submit("list_files", {}, timeout_seconds=0.01)
     with pytest.raises(RuntimeError, match="uncertain"):
-        client.submit("list_files", {})
+        client.submit("stop", {})
 
 
 def test_serve_retries_result_without_reexecuting(tmp_path):
@@ -96,7 +96,7 @@ def test_serve_retries_result_without_reexecuting(tmp_path):
             assert json.loads(data)["stopped"]
 
     transport = Transport()
-    serve(executor, "command", "result", transport=transport, deadline_seconds=0.2, poll_seconds=0.001)
+    serve(executor, "command", "result", transport=transport, deadline_seconds=1, poll_seconds=0.001)
     assert transport.writes == 2
 
 
@@ -316,3 +316,123 @@ def test_guard_io_failure_after_child_exit_is_not_success(tmp_path, monkeypatch)
     assert result["phase"] == "scratch guard"
     assert result["exit_code"] is not None
     assert "cache cannot be read" in result["reason"]
+
+
+@pytest.mark.parametrize("side", ["host", "task"])
+@pytest.mark.parametrize("method", ["put", "get"])
+@pytest.mark.parametrize("error", ["timeout", "connection", "503"])
+def test_transient_mailbox_failures_retry_same_sequence(tmp_path, monkeypatch, side, method, error):
+    import urllib.error
+
+    from openultrasast.search import executor as module
+
+    now = [0.0]
+    monkeypatch.setattr(module.time, "monotonic", lambda: now[0])
+    monkeypatch.setattr(module.time, "sleep", lambda delay: now.__setitem__(0, now[0] + delay))
+    app = InProcessExecutor(tmp_path)
+    failures, payloads, results = [], [], []
+
+    def fail():
+        failures.append(now[0])
+        if len(failures) <= 4:
+            if error == "503":
+                raise urllib.error.HTTPError("https://store.example", 503, "unavailable", {}, None)
+            raise (TimeoutError if error == "timeout" else ConnectionError)("temporary store failure")
+
+    class Flaky:
+        def put(self, url, data, timeout):
+            payloads.append(data)
+            if side == "host":
+                results[:] = [app.execute(Command(**json.loads(data)))]
+            if method == "put":
+                fail()
+
+        def get(self, url, timeout):
+            if method == "get":
+                fail()
+            if side == "host":
+                return json.dumps(results[-1]).encode()
+            return Command(1, "stop", {}, 1).validate()
+
+    if side == "host":
+        client = ObjectStoreExecutor("https://store.example/command", "https://store.example/result", transport=Flaky())
+        assert client.submit("stop", {}, timeout_seconds=1)["stopped"]
+        assert client.seq == 1 and not client.uncertain
+    else:
+        serve(app, "command", "result", transport=Flaky(), deadline_seconds=100)
+    assert app.seq == 1 and app.stopped
+    assert len(set(payloads)) == 1
+    assert len(failures) == 5 and now[0] > 1
+
+
+def test_retry_budget_is_90_seconds_and_permanent_http_errors_fail_fast(monkeypatch):
+    import urllib.error
+
+    from openultrasast.search import executor as module
+
+    now, calls = [0.0], []
+    monkeypatch.setattr(module.time, "monotonic", lambda: now[0])
+    monkeypatch.setattr(module.time, "sleep", lambda delay: now.__setitem__(0, now[0] + delay))
+
+    def fail(timeout):
+        calls.append(timeout)
+        raise TimeoutError("still unavailable")
+
+    with pytest.raises(TimeoutError):
+        module.retry_mailbox(fail)
+    assert now[0] == 90 and all(0 < value <= 10 for value in calls)
+
+    def forbidden(timeout):
+        raise urllib.error.HTTPError("https://store.example", 403, "forbidden", {}, None)
+
+    with pytest.raises(urllib.error.HTTPError):
+        module.retry_mailbox(forbidden)
+    assert now[0] == 90
+
+
+def test_uncertain_command_can_be_resubmitted_without_changing_seq(tmp_path):
+    class Lost(Mailbox):
+        available = False
+
+        def put(self, url, data, timeout):
+            self.writes += 1
+            self.result = json.dumps(self.executor.execute(Command(**json.loads(data)))).encode()
+
+        def get(self, url, timeout):
+            return self.result if self.available else None
+
+    (tmp_path / "input").write_text("original")
+    mailbox = Lost(InProcessExecutor(tmp_path))
+    client = ObjectStoreExecutor("https://store.example/command", "https://store.example/result", transport=mailbox, poll_seconds=0.001)
+    with pytest.raises(TimeoutError):
+        client.submit("read_file", {"path": "input"}, timeout_seconds=0.01)
+    (tmp_path / "input").write_text("changed")
+    mailbox.available = True
+    assert client.submit("read_file", {"path": "input"})["text"] == "original"
+    assert client.seq == 1
+
+
+def test_url_transport_uploads_file_without_buffering(tmp_path):
+    from contextlib import nullcontext
+
+    from openultrasast.search.executor import URLTransport
+
+    path = tmp_path / "archive"
+    path.write_bytes(b"synthetic archive" * 10000)
+    transport = URLTransport()
+    seen = {}
+
+    class Opener:
+        def open(self, request, timeout):
+            assert request.get_method() == "PUT"
+            assert not isinstance(request.data, bytes)
+            seen["length"] = int(request.get_header("Content-length"))
+            seen["bytes"] = 0
+            while block := request.data.read(8192):
+                seen["bytes"] += len(block)
+            return nullcontext()
+
+    transport.opener = Opener()
+    with path.open("rb") as stream:
+        transport.put("https://files.example/archive", stream, 10)
+    assert seen["length"] == seen["bytes"] == path.stat().st_size

tests/test_search_pilot_run.py

@@ -131,7 +131,7 @@ def test_worker_failure_is_not_no_evidence(tmp_path):
     record = pilot.run()
     assert record["outcome"] == "inconclusive"
     assert record["failures"][0]["phase"] == "explore"
-    assert "ValueError: explore must use tools or finish" in record["failures"][0]["reason"]
+    assert "second plain-text reply:" in record["failures"][0]["reason"]
 
 
 def test_shared_budget_reserved_and_settled_on_error(tmp_path):
@@ -500,3 +500,41 @@ def test_pilot_retains_command_diagnostics(tmp_path, phase):
     assert "line-29" in failure["stderr"] and "line-0\n" not in failure["stderr"]
     assert "hidden" not in failure["stderr"] and "https://" not in failure["stderr"]
     assert len(failure["stderr"]) <= 1500
+
+
+@pytest.mark.parametrize("failure_at", ["acquire", "body"])
+def test_pilot_executor_deletes_task_when_submit_times_out(tmp_path, monkeypatch, failure_at):
+    import time
+    from types import SimpleNamespace
+
+    from openultrasast.search.executor import ObjectStoreExecutor
+
+    pilot = make_pilot(tmp_path)
+    deleted, calls = [], []
+    store = SimpleNamespace(
+        _put=lambda *a: None,
+        _delete=deleted.append,
+        presign_get=lambda key, expires: "https://files.example/" + key,
+        presign_put=lambda key, expires: "https://files.example/" + key,
+    )
+    pilot.lane = SimpleNamespace(
+        store=store,
+        host="files.example",
+        args=SimpleNamespace(atespace="default"),
+        workload=SimpleNamespace(image="registry/task@sha256:" + "a" * 64),
+        ax=lambda *a, **kw: calls.append((a, kw)),
+        clock=time.monotonic,
+        pause=lambda _: None,
+    )
+
+    def submit(self, name, args, **kwargs):
+        if failure_at == "acquire" or name == "list_files":
+            raise TimeoutError("executor command upload deadline")
+        return {"exit_code": 0, "input_bytes": 10}
+
+    monkeypatch.setattr(ObjectStoreExecutor, "submit", submit)
+    with pytest.raises(TimeoutError, match="upload deadline"), pilot.executor("a" * 40, deadline=60) as session:
+        session.client.submit("list_files", {})
+    assert calls[-1][0][:2] == ("delete", "task")
+    assert len(deleted) == 4
+    assert pilot.record["executor_task_metrics"][-1]["end"] == "execution_failure"

tests/test_search_task_storage.py

@@ -75,7 +75,9 @@ def test_executor_exports_built_input_and_verifier_needs_no_build(workspace, mon
     blobs = []
     app = executor.InProcessExecutor(repo, task_boundary=True)
     app.prepared_put_url = "https://files.example/output"
-    monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
+    monkeypatch.setattr(
+        executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data.read() if hasattr(data, "read") else data)
+    )
     result = app.submit("prepare_verification", spec())
     assert result["status"] == "ok", result
     destination = workspace / "unpacked"
@@ -140,7 +142,9 @@ def test_preparation_builds_before_export_and_exports_products(workspace, monkey
     app = executor.InProcessExecutor(repo, task_boundary=True)
     app.prepared_put_url = "https://files.example/output"
     blobs, commands = [], []
-    monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
+    monkeypatch.setattr(
+        executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data.read() if hasattr(data, "read") else data)
+    )
 
     def build(argv, timeout, limits, **kwargs):
         commands.append(argv)
@@ -246,7 +250,9 @@ def test_preparation_uses_project_gradle_wrapper(workspace, monkeypatch, project
     app = executor.InProcessExecutor(repo, task_boundary=True)
     app.prepared_put_url = "https://files.example/output"
     blobs = []
-    monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
+    monkeypatch.setattr(
+        executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data.read() if hasattr(data, "read") else data)
+    )
     request = spec()
     request["demo"]["build"] = {"recipe": "gradle", "arguments": [project]}
     result = app.submit("prepare_verification", request)
@@ -302,7 +308,9 @@ def test_prepare_build_virtual_memory(workspace, monkeypatch, memory_bytes):
     app = executor.InProcessExecutor(repo, task_boundary=True)
     app.prepared_put_url = "https://files.example/output"
     blobs = []
-    monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
+    monkeypatch.setattr(
+        executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data.read() if hasattr(data, "read") else data)
+    )
     request = spec()
     request["demo"]["build"] = {"recipe": "npm", "arguments": ["."]}
     limits = {} if memory_bytes is None else {"memory_bytes": memory_bytes}
@@ -440,7 +448,9 @@ def test_prepare_preserves_large_config_and_task_verifier_applies_it(workspace,
     blobs = []
     app = executor.InProcessExecutor(repo, task_boundary=True)
     app.prepared_put_url = "https://files.example/output"
-    monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
+    monkeypatch.setattr(
+        executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data.read() if hasattr(data, "read") else data)
+    )
     result = app.submit("prepare_verification", request)
     assert result["status"] == "ok", result
     destination = workspace / "unpacked"
@@ -454,3 +464,64 @@ def test_prepare_preserves_large_config_and_task_verifier_applies_it(workspace,
     observation = verify_side_task(Side(destination / "checkout", products=destination / "products"), demo, "path")
     assert observation.outcome == "observed", observation
     assert len(observation.runs) == 1
+
+
+def test_large_export_spools_to_workspace_and_excludes_caches(workspace, monkeypatch):
+    import hashlib
+    import tarfile
+
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "app.py").write_text("print('safe')")
+    # Incompressible input exceeds the former 64 MiB archive cap.
+    with (repo / "large.bin").open("wb") as stream:
+        for _ in range(129):
+            stream.write(os.urandom(1024**2))
+    for directory in [".npm", ".pip", ".m2", ".composer", ".gradle", "__pycache__", ".git", "node_modules/.cache"]:
+        path = repo / "nested" / directory
+        path.mkdir(parents=True)
+        (path / "excluded").write_bytes(b"cache")
+    seen = {}
+
+    def upload(self, url, data, timeout):
+        assert not isinstance(data, bytes)
+        assert str(workspace) in os.readlink(f"/proc/self/fd/{data.fileno()}")
+        seen["size"] = os.fstat(data.fileno()).st_size
+        seen["sha256"] = hashlib.file_digest(data, "sha256").hexdigest()
+        data.seek(0)
+        with tarfile.open(fileobj=data, mode="r:gz") as archive:
+            names = archive.getnames()
+            assert "checkout/large.bin" in names
+            assert not any(name.endswith("excluded") for name in names)
+        data.seek(0)
+
+    monkeypatch.setattr(executor.URLTransport, "put", upload)
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    result = app.submit("prepare_verification", spec(), timeout_seconds=60)
+    assert result["status"] == "ok", result
+    assert 128 * 1024**2 < result["archive_bytes"] == seen["size"]
+    assert result["sha256"] == seen["sha256"]
+    assert verify_task.MAX_ARCHIVE_BYTES == 1024**3
+
+
+def test_export_over_cap_reports_size_and_five_relative_directories(workspace, monkeypatch):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "app.py").write_text("print('safe')")
+    for i in range(6):
+        directory = repo / f"dir{i}"
+        directory.mkdir()
+        (directory / "data").write_bytes(os.urandom(1000 * (i + 1)))
+    monkeypatch.setattr(verify_task, "MAX_ARCHIVE_BYTES", 1000)
+    monkeypatch.setattr(executor.URLTransport, "put", lambda *a: pytest.fail("oversized upload"))
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    result = app.submit("prepare_verification", spec())
+    assert result["status"] == "could_not_build"
+    assert str(result["archive_bytes"]) + " bytes > 1000 bytes" in result["reason"]
+    assert str(workspace) not in result["reason"]
+    assert "top 5 directories: checkout (" in result["reason"]
+    for i in range(2, 6):
+        assert f"checkout/dir{i} (" in result["reason"]
+    assert "dir0" not in result["reason"] and "dir1" not in result["reason"]

tests/test_search_verify_transport.py

@@ -77,7 +77,12 @@ def test_entry_reads_input_and_scrubs_environment(tmp_path, monkeypatch, capsys)
     (bundle / "products").mkdir()
     (bundle / "spec.json").write_text(json.dumps({"demo": demo(), "family": "injection", "timeout_seconds": 2}))
     blob = entry.pack_checkout(bundle)
-    monkeypatch.setattr(entry, "download", lambda url, limit: blob)
+
+    def download(url, limit):
+        assert limit == 1024**3
+        return blob
+
+    monkeypatch.setattr(entry, "download", download)
     outputs = []
     monkeypatch.setattr(entry, "upload", lambda url, data: outputs.append(json.loads(data)))
 

tests/test_search_worker.py

@@ -457,3 +457,27 @@ def test_finish_wins_over_other_calls_without_executing_them(tmp_path, finish_fi
     assert result["fact"]["text"] == "Confirmed app source"
     assert worker.metrics["warnings"] == 1
     assert worker.evidence == {"tool:1"}
+
+
+def test_plain_text_gets_one_reminder_then_can_finish(tmp_path):
+    model = StubModel([{"content": "I will inspect"}, tool("finish", failure="no evidence")])
+    result = Worker(model, repo=tmp_path, demo=tmp_path / "demo", max_steps=2)(task("explore"))
+    assert result["failure"] == "no evidence"
+    assert len(model.calls) == 2
+    assert model.calls[1]["messages"][-1] == {"role": "user", "content": "Use a tool or call finish; plain text is not accepted."}
+
+
+def test_second_plain_text_records_first_300_characters(tmp_path):
+    text = "second response " * 40
+    model = StubModel([{"content": "first"}, {"content": text}])
+    result = Worker(model, repo=tmp_path, demo=tmp_path / "demo", max_steps=3)(task("explore"))
+    assert result["status"] == "execution_failure"
+    assert result["failure"] == "second plain-text reply: " + text[:300]
+    assert len(model.calls) == 2
+
+
+def test_plain_text_consumes_step(tmp_path):
+    model = StubModel([{"content": "first"}])
+    result = Worker(model, repo=tmp_path, demo=tmp_path / "demo", max_steps=1)(task("explore"))
+    assert result["end_detail"] == "worker_steps"
+    assert len(model.calls) == 1