search-with-proof: cluster-unavailable detection, kept messages, run-level abort

Zero running workers behind "no free workers available" now raises
ClusterUnavailable instead of waiting out the capacity deadline; CapacityDeadline
carries its message; cleanup failures no longer mask the original exception; the
pilot driver aborts the run on infrastructure failure before any model call.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-05 20:26:37 +02:00
co-authored by Claude Fable 5.1
parent e39a4882b0
commit 4be2063d40
7 changed files with 284 additions and 16 deletions
+18
View File
@@ -222,3 +222,21 @@ def test_scratch_failure_preserves_exit_and_stderr(workspace, monkeypatch):
assert result["exit_code"] is not None
assert result["stderr"] == "pip build detail"
assert result["reason"].startswith("ScratchLimit:")
def test_capacity_deadline_plain_diagnostic_survives():
from benchmarks.ax.batch import CapacityDeadline
message = "no free workers available before deadline"
assert task_storage.diagnostic(message) == message
assert task_storage.exception_reason(CapacityDeadline(message)) == "CapacityDeadline: " + message
def test_cleanup_diagnostic_retains_both_causes():
error = ValueError("original apply failed")
error.executor_cleanup_reason = "executor cleanup failed: " + "details " * 100 + "delete failed"
error.args = (str(error) + "; " + error.executor_cleanup_reason,)
reason = task_storage.exception_reason(error)
assert reason.startswith("ValueError: original apply failed;")
assert reason.endswith("delete failed")
assert len(reason) <= 300