marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge(search-with-proof): cluster-unavailable handling [skip ci]

Browse Source 
No image rebuild: VM-side changes only (operator asked to stop per-merge rebuilds).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

# Conflicts:
#	benchmarks/ax/batch.py
This commit is contained in:
Marius Ciepluch
2026-10-05 20:27:01 +02:00
parent 7fb061fb69 4be2063d40
commit daf95b16a1
7 changed files with 280 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/ax/batch.py
+47 -7
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import hashlib
import ipaddress
import json
import logging
import os
import queue
import random
@@ -40,6 +41,10 @@ class NoFreeWorkers(Exception):
"""Capacity back-pressure, never a sandbox or instrument failure."""
class ClusterUnavailable(Exception):
"""No running workers were observed in the worker namespace."""
class CapacityDeadline(Exception):
"""No worker became available before the task deadline."""
@@ -51,6 +56,7 @@ AX_SLOTS = threading.BoundedSemaphore(2)
def apply_when_available(lane, document, end):
started = lane.clock()
next_health_check = started + 60
while lane.clock() < end:
try:
lane.ax("apply", "-f", "-", manifest=document, timeout=min(30, end - lane.clock()))
@@ -62,6 +68,9 @@ def apply_when_available(lane, document, end):
lane.ax("delete", "task", document["metadata"]["name"])
except Exception as exc:
raise RuntimeError("capacity cleanup failed: " + exception_reason(exc)) from None
if lane.clock() >= next_health_check:
lane.check_cluster_health(timeout=min(10, max(0.01, end - lane.clock())))
next_health_check = lane.clock() + 60
# Substrate's golden-snapshot copy holds the other worker for about a second after each new task
# (operator, 2026-10-05): retry quickly for the first minute, then back off while real tasks run.
delay = random.uniform(2, 5) if lane.clock() - started < 60 else random.uniform(15, 30)
@@ -163,7 +172,8 @@ def bounded_read(read, timeout):
class AXLane:
def __init__(self, args, workload, store=None, pause=time.sleep, clock=time.monotonic):
def __init__(self, args, workload, store=None, pause=time.sleep, clock=time.monotonic, runner=None):
self.runner = runner or subprocess.run
self.workload = workload
self.args = args
self.pause, self.clock = pause, clock
@@ -204,11 +214,35 @@ class AXLane:
record = json.loads((output / "result.json").read_text())
return {**record, "worker_ip": worker, "ax_attempts": attempts}
def check_cluster_health(self, *, timeout=10):
env = dict(os.environ)
if self.args.kubeconfig:
env["KUBECONFIG"] = str(Path(self.args.kubeconfig).expanduser())
done = self.runner(
["kubectl", "-n", "ax-workers", "get", "pods", "-o", "json"],
env=env,
capture_output=True,
text=True,
timeout=timeout,
)
if done.returncode:
raise RuntimeError(f"worker pod health check failed (exit {done.returncode})")
try:
pods = json.loads(done.stdout)["items"]
if not isinstance(pods, list):
raise ValueError("invalid pod list")
phases = [pod["status"]["phase"] for pod in pods]
except (ValueError, KeyError, TypeError):
raise RuntimeError("worker pod health check returned invalid pod data") from None
running = phases.count("Running")
if not running:
raise ClusterUnavailable(f"worker pods: total={len(pods)}, running=0, pending={phases.count('Pending')}")
def ax(self, *args, manifest=None, timeout=30):
env = dict(os.environ)
if self.args.kubeconfig:
env["KUBECONFIG"] = str(Path(self.args.kubeconfig).expanduser())
done = subprocess.run(
done = self.runner(
[str(Path(self.args.ax_bin).expanduser()), *args, *(["--atespace", self.args.atespace] if args[0] != "apply" else [])],
input=json.dumps(manifest) if manifest is not None else None,
env=env,
@@ -632,7 +666,7 @@ class SearchExecutorTask:
self.end = time.monotonic() + self.deadline
if not isinstance(self.lane, DockerLane):
if not AX_SLOTS.acquire(timeout=self.deadline):
raise CapacityDeadline
raise CapacityDeadline("no free workers available before deadline")
self.slot = True
expires = timedelta(seconds=self.deadline + 600)
store = self.lane.store
@@ -677,8 +711,8 @@ class SearchExecutorTask:
raise
self.client = ObjectStoreExecutor(command_put, result_get)
return self
except Exception:
self.__exit__(None, None, None)
except Exception as exc:
self.__exit__(type(exc), exc, exc.__traceback__)
raise
def prepare(self, spec):
@@ -692,7 +726,7 @@ class SearchExecutorTask:
raise ValueError("executor archive missing or digest mismatch")
return stored[0]
def __exit__(self, *exc):
def __exit__(self, exc_type, exc, traceback):
errors = []
if self.submitted:
try:
@@ -708,4 +742,10 @@ class SearchExecutorTask:
AX_SLOTS.release()
self.slot = False
if errors:
raise RuntimeError("executor cleanup failed: " + ", ".join(errors))
reason = "executor cleanup failed: " + ", ".join(errors)
logging.getLogger(__name__).error("%s", reason)
if exc is not None:
exc.executor_cleanup_reason = reason
exc.args = (str(exc) + "; " + reason,)
return False
raise RuntimeError(reason)
benchmarks/search/pilot_run.py
+26 -7
View File
@@ -17,7 +17,7 @@ from contextlib import contextmanager
from dataclasses import replace
from pathlib import Path
from benchmarks.ax.batch import AXLane, SearchExecutorTask, Workload, validate_image
from benchmarks.ax.batch import AXLane, ClusterUnavailable, SearchExecutorTask, Workload, validate_image
from benchmarks.ax.search_verify import AXSideDispatcher, validate_result
from benchmarks.search.executor_smoke import CLONE
from openultrasast.config import load_dotenv
@@ -65,6 +65,10 @@ def scripted(schema):
)
class RepositoryFailure(ValueError):
"""The selected checkout or candidate cannot be read."""
class Pilot:
def __init__(self, args, row, index, root, store, run_budget, client, prices, lane=None):
self.args, self.root, self.lane = args, root, lane
@@ -156,14 +160,14 @@ class Pilot:
)
if result.get("exit_code") != 0 or result.get("error"):
self.failure("acquisition", "checkout_command_failed")
raise ValueError("checkout failed")
raise RepositoryFailure("checkout failed")
remaining = deadline - (time.monotonic() - acquired_at)
if remaining < 1:
raise BudgetExhausted("acquisition deadline", usd=0, calls=0)
result = task.client.submit("read_file", {"path": self.row["file"]}, timeout_seconds=min(30, remaining))
if not isinstance(result.get("input_bytes"), int) or result["input_bytes"] <= 0:
self.failure("acquisition", "candidate_input_unreadable")
raise ValueError("unreadable input")
raise RepositoryFailure("unreadable input")
self.record["input_bytes"] += result["input_bytes"]
yield task
@@ -281,6 +285,8 @@ class Pilot:
raise BudgetExhausted("run ceiling", usd=0, calls=0)
if result.get("status") != "ok":
self.failure(task.step, result.get("failure", "worker_execution_failed"))
if self.worker.metrics["model_calls"] == 0:
raise RuntimeError(self.record["failures"][-1]["reason"])
if not self.verifications:
self.record["outcome"] = "inconclusive"
return result
@@ -304,6 +310,8 @@ class Pilot:
else "inconclusive"
)
self.record["end_reason"] = "instrument_failure"
if isinstance(exc, ClusterUnavailable) or (self.worker.metrics["model_calls"] == 0 and not isinstance(exc, RepositoryFailure)):
self.record["aborted_reason"] = self.record["failures"][-1]["reason"]
metrics = self.worker.metrics
self.record.update(
spend={k: metrics[k] for k in ("reserved", "settled")},
@@ -425,15 +433,20 @@ def main(argv=None):
client = OpenRouterChatClient(
api_key=os.environ[DEEPSEEK_KEY_ENV], base_url=os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL), max_attempts=1
)
lane = AXLane(args, Workload(args.image, frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}), 900, validate_result))
root = args.private_root / uuid.uuid4().hex
root.mkdir(parents=True, mode=0o700)
store = open_store(args.board_memory) if args.board_memory else FileStore(root / "boards")
records = []
started = time.monotonic()
args.out.parent.mkdir(parents=True, exist_ok=True)
with args.out.open("x") as output:
for index in indices:
aborted_reason = None
try:
store = open_store(args.board_memory) if args.board_memory else FileStore(root / "boards")
if not args.dry_run:
lane = AXLane(args, Workload(args.image, frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}), 900, validate_result))
except Exception as exc:
aborted_reason = exception_reason(exc)
for index in indices if aborted_reason is None else []:
local = root / str(index)
local.mkdir()
if args.dry_run:
@@ -447,8 +460,14 @@ def main(argv=None):
records.append(record)
output.write(json.dumps(record, allow_nan=False) + "\n")
output.flush()
if record.get("aborted_reason"):
aborted_reason = record["aborted_reason"]
break
summary = dict(
type="summary",
end_reason="instrument_failure" if aborted_reason else "completed",
aborted_reason=aborted_reason,
skipped_pairs=indices[len(records) :],
searches=len(records),
dry_run=args.dry_run,
side=args.side,
@@ -462,7 +481,7 @@ def main(argv=None):
wall_seconds=time.monotonic() - started,
)
output.write(json.dumps(summary, allow_nan=False) + "\n")
return int(any(r["end_reason"] == "instrument_failure" for r in records))
return int(bool(aborted_reason) or any(r["end_reason"] == "instrument_failure" for r in records))
if __name__ == "__main__":