search-with-proof: name the exhausted limit, keep partial evidence, tighter reservation

budget_spent carries end_detail (spend, task_spend, reason_rounds, ...); a task
whose allowance cannot cover the next call finishes with its facts; per-task
metrics in pilot records; input reservation bytes/2 + 1024 per message (measured
3-4x over-reservation before).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/test_search_coordinator.py

@@ -71,7 +71,7 @@ def test_claim_checkpoint_survives_interrupt_without_free_spend(tmp_path):
     checkpoint = b.head
     resumed = Board.resume(b.store, "search-1", checkpoint, coordinator="host")
     state = Coordinator(resumed, lambda task: {"status": "ok", "cost_usd": 0, "intents": []}).run()
-    assert state["checkpoint"]["spent_usd"] == 0.05
+    assert state["checkpoint"]["spent_usd"] == SearchBudget().max_call_usd
     assert state["checkpoint"]["tasks"] == 3
 
 
@@ -98,7 +98,7 @@ def test_cleanup_failure_halts_and_charges_unknown_spend(tmp_path):
     with pytest.raises(RuntimeError, match="cleanup"):
         Coordinator(b, broken).run()
     assert len(calls) == 1
-    assert b.state["checkpoint"]["spent_usd"] == 0.05
+    assert b.state["checkpoint"]["spent_usd"] == SearchBudget().max_call_usd
     assert b.state["checkpoint"]["pending"]["id"] == calls[0].id
 
 
@@ -164,3 +164,56 @@ def test_invalid_identifiers_refused_before_paths(tmp_path):
             Board(FileStore(tmp_path), bad, coordinator="host")
         with pytest.raises(ValueError):
             SearchTask(bad, "search-1", "reason", "", {}, asdict(SearchBudget()))
+
+
+@pytest.mark.parametrize("detail", ["task_spend", "worker_steps", "worker_wall", "run_ceiling"])
+def test_worker_limit_preserves_fact_and_survives_resume(tmp_path, detail):
+    def execute(task):
+        if task.step == "reason":
+            return {"status": "ok", "cost_usd": 0, "intents": [{"description": "inspect"}]}
+        return {
+            "status": "ok",
+            "cost_usd": 0.01,
+            "budget_exhaustion": {"end_detail": detail, "usd": 0.01, "calls": 1},
+            "fact": {"text": "Confirmed partial observation", "evidence_refs": ["tool:1"]},
+        }
+
+    b = board(tmp_path)
+    state = Coordinator(b, execute).run()
+    assert state["end_reason"] == "budget_spent" and state["end_detail"] == detail
+    assert state["facts"][0]["text"] == "Confirmed partial observation"
+    assert state["intents"][0]["status"] == "concluded"
+    assert state["checkpoint"]["spent_usd"] == 0.01
+    assert Board.resume(b.store, "search-1", b.head, coordinator="host").state == state
+
+
+@pytest.mark.parametrize("detail", ["spend", "tasks", "reason_rounds", "demonstrations", "task_wall", "search_wall", "run_ceiling"])
+def test_coordinator_identifies_each_own_limit(tmp_path, detail):
+    from openultrasast.plane.budget import SpendBudget
+
+    now = [0]
+
+    def execute(task):
+        if detail in {"task_wall", "search_wall"}:
+            now[0] = 1000 if detail == "task_wall" else 20000
+        if task.step == "reason":
+            return {"status": "ok", "cost_usd": 0, "intents": [{"description": str(task.id)}]}
+        if task.step == "explore":
+            return {"status": "ok", "cost_usd": 0, "fact": {"text": "candidate", "evidence_refs": ["tool:1"], "demo": "demo/"}}
+        return {"status": "ok", "cost_usd": 0, "outcome": "inconclusive"}
+
+    limits = {
+        "spend": {"spend_usd": 0},
+        "tasks": {"tasks": 1},
+        "reason_rounds": {"reason_rounds": 1},
+        "demonstrations": {"demonstrations": 1},
+    }
+    state = Coordinator(
+        board(tmp_path),
+        execute,
+        budget=SearchBudget(**limits.get(detail, {})),
+        clock=lambda: now[0],
+        push_budget=SpendBudget(0) if detail == "run_ceiling" else None,
+    ).run()
+    assert state["end_reason"] == "budget_spent"
+    assert state["end_detail"] == detail

tests/test_search_pilot_run.py

@@ -63,6 +63,8 @@ def test_ceiling_refuses_before_stub_call(tmp_path):
     record = json.loads(output.read_text().splitlines()[0])
     assert record["model_calls"] == 0
     assert record["end_reason"] == "budget_spent"
+    assert record["end_detail"] == "run_ceiling"
+    assert record["task_metrics"][0]["end"] == "run_ceiling"
 
 
 def make_pilot(tmp_path, *, side="fixed", dry_run=False):
@@ -221,6 +223,13 @@ def test_live_wiring_with_fake_mailbox_no_network(tmp_path, monkeypatch, verify_
     assert record["end_reason"] == "goal_met"
     assert record["model_calls"] == 3
     assert record["executor_tasks"] == 3
+    assert len(record["executor_task_metrics"]) == 3
+    assert sum(t["model_calls"] for t in record["executor_task_metrics"]) == 3
+    for row in record["executor_task_metrics"]:
+        assert row["reserved"] >= row["settled"]
+        assert row["wall_seconds"] > 0
+        assert row["end"] == "ok"
+        assert {"tokens_in", "tokens_out"} <= row.keys()
     assert record["verify_tasks"] == (6 if verify_lane == "ax" else 0)
     assert record["tasks_submitted"] == (9 if verify_lane == "ax" else 3)
     assert record["input_bytes"] > 0

tests/test_search_task_storage.py

@@ -2,7 +2,9 @@
 
 import json
 import os
+import signal
 import sys
+import time
 from pathlib import Path
 
 import pytest
@@ -224,6 +226,63 @@ def test_scratch_failure_preserves_exit_and_stderr(workspace, monkeypatch):
     assert result["reason"].startswith("ScratchLimit:")
 
 
+@pytest.mark.parametrize("project", [".", "subproject"])
+def test_preparation_uses_project_gradle_wrapper(workspace, monkeypatch, project):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    root = repo / project
+    root.mkdir(exist_ok=True)
+    # A real local child proves wrapper execution without system Gradle or network.
+    wrapper = root / "gradlew"
+    wrapper.write_text(
+        'test "$1" = "--no-daemon" || exit 1\n'
+        'test "$2" = "--project-dir" || exit 2\n'
+        'test "$4" = "assemble" || exit 3\n'
+        'mkdir -p "$3/build"\n'
+        'printf compiled > "$3/build/product"\n'
+    )
+    wrapper.chmod(0o644)
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    blobs = []
+    monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
+    request = spec()
+    request["demo"]["build"] = {"recipe": "gradle", "arguments": [project]}
+    result = app.submit("prepare_verification", request)
+    assert result["status"] == "ok", result
+    unpacked = workspace / "unpacked"
+    verify_task.extract_checkout(blobs[0], unpacked)
+    assert (unpacked / "checkout" / project / "build/product").read_text() == "compiled"
+    assert json.loads((unpacked / "spec.json").read_text())["demo"]["build"]["recipe"] == "none"
+
+
+def test_preparation_with_many_host_processes(workspace, monkeypatch):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "gradlew").write_text('mkdir -p "$3/build"\nprintf compiled > "$3/build/product"\n')
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: None)
+    request = spec()
+    request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]}
+    children = []
+    try:
+        for _ in range(300):
+            pid = os.fork()
+            if pid == 0:
+                time.sleep(60)
+                os._exit(0)
+            children.append(pid)
+        result = app.submit("prepare_verification", request)
+        assert result["status"] == "ok", result
+        assert (repo / "build/product").read_text() == "compiled"
+    finally:
+        for pid in children:
+            os.kill(pid, signal.SIGKILL)
+        for pid in children:
+            os.waitpid(pid, 0)
+
+
 def test_capacity_deadline_plain_diagnostic_survives():
     from benchmarks.ax.batch import CapacityDeadline
 

tests/test_search_worker.py

@@ -307,3 +307,82 @@ def test_prompt_contracts_and_demo_example(tmp_path, step):
         assert len(prompt) / 4 < 1000  # rough offline token estimate, not a tokenizer
     else:
         assert "file:line" in prompt and "final call for finish" in prompt
+
+
+@pytest.mark.parametrize("with_evidence", [False, True])
+def test_task_spend_finishes_with_confirmed_observations(tmp_path, with_evidence):
+    from openultrasast.model.endpoint import Prices
+
+    (tmp_path / "input.txt").write_text("confirmed input\n" * 400)
+
+    class UsageStub(StubModel):
+        def complete_chat_raw(self, **kwargs):
+            raw = super().complete_chat_raw(**kwargs)
+            raw["usage"] = {"prompt_tokens": 100, "completion_tokens": 100}
+            return raw
+
+    model = UsageStub([tool("read_file", path="input.txt")])
+    worker = Worker(model, repo=tmp_path, demo=tmp_path / "demo", prices=Prices(1, 1, 1), executor=InProcessExecutor(tmp_path))
+    job = task("explore")
+    # First reservation fits; the second includes the real file contents and cannot fit.
+    job.limits["max_call_usd"] = 0.01 if with_evidence else 0.000001
+    result = worker(job)
+    assert result["end_detail"] == result["budget_exhaustion"]["end_detail"] == "task_spend"
+    assert len(model.calls) == int(with_evidence)
+    assert worker.metrics["reserved"] >= worker.metrics["settled"]
+    if with_evidence:
+        assert result["status"] == "ok"
+        assert "confirmed input" in result["fact"]["text"]
+        assert result["fact"]["evidence_refs"] == ["tool:1"]
+        assert result["cost_usd"] == pytest.approx(0.0002)
+    else:
+        assert result["status"] == "execution_failure"
+        assert "task_spend" in result["failure"]
+        assert "fact" not in result
+
+
+def test_calibrated_reservation_covers_reported_usage(tmp_path):
+    import math
+
+    from openultrasast.model.endpoint import Prices
+    from openultrasast.plane.budget import SpendBudget
+
+    class UsageStub(StubModel):
+        def complete_chat_raw(self, **kwargs):
+            raw = super().complete_chat_raw(**kwargs)
+            raw["usage"] = {"prompt_tokens": 27000, "completion_tokens": 1024}
+            return raw
+
+    model = UsageStub([reply({})])
+    worker = Worker(model, repo=tmp_path, demo=tmp_path / "demo", prices=Prices(1, 1, 2))
+    worker.reason("facts: []\nnote: " + "x" * 85000, SpendBudget(1))
+    call = model.calls[0]
+    expected_input = math.ceil(len(json.dumps([call["messages"], call["tools"]], ensure_ascii=False).encode()) / 2) + 1024 * (
+        len(call["messages"]) + 1
+    )
+    assert worker.metrics["reserved"] == pytest.approx((expected_input + 1024 * 2) / 1_000_000)
+    assert worker.metrics["reserved"] >= worker.metrics["settled"] == pytest.approx(0.029048)
+
+
+@pytest.mark.parametrize("detail", ["worker_steps", "worker_wall", "run_ceiling"])
+def test_worker_exhaustion_is_structured(tmp_path, monkeypatch, detail):
+    from openultrasast.model.endpoint import Prices
+    from openultrasast.plane.budget import SpendBudget
+
+    worker = Worker(
+        StubModel([tool("list_files")]),
+        repo=tmp_path,
+        demo=tmp_path / "demo",
+        executor=InProcessExecutor(tmp_path),
+        max_steps=1,
+        prices=Prices(1, 1, 1),
+        run_budget=SpendBudget(0 if detail == "run_ceiling" else 1),
+    )
+    if detail == "worker_wall":
+        from openultrasast.search import worker as module
+
+        times = iter([0, 901])
+        monkeypatch.setattr(module.time, "monotonic", lambda: next(times))
+    result = worker(task("explore"))
+    assert result["budget_exhaustion"]["end_detail"] == detail
+    assert result["end_detail"] == detail