From 89516d4000a7443e4ff65f77f095c0915e0b5fe3 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:07:47 +0200 Subject: [PATCH] ax batch: fast capacity retries in the first minute (golden-snapshot window) Operator 2026-10-05: Substrate boots a ~1 s golden-snapshot actor per new task on the other worker; retry apply after 2-5 s for the first minute, then 15-30 s. Co-Authored-By: Claude Opus 5.5 --- benchmarks/ax/batch.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/ax/batch.py b/benchmarks/ax/batch.py index 81e3233..24708e0 100644 --- a/benchmarks/ax/batch.py +++ b/benchmarks/ax/batch.py @@ -50,6 +50,7 @@ AX_SLOTS = threading.BoundedSemaphore(2) def apply_when_available(lane, document, end): + started = lane.clock() while lane.clock() < end: try: lane.ax("apply", "-f", "-", manifest=document, timeout=min(30, end - lane.clock())) @@ -61,7 +62,10 @@ def apply_when_available(lane, document, end): lane.ax("delete", "task", document["metadata"]["name"]) except Exception as exc: raise RuntimeError("capacity cleanup failed: " + exception_reason(exc)) from None - lane.pause(min(random.uniform(15, 30), max(0, end - lane.clock()))) + # Substrate's golden-snapshot copy holds the other worker for about a second after each new task + # (operator, 2026-10-05): retry quickly for the first minute, then back off while real tasks run. + delay = random.uniform(2, 5) if lane.clock() - started < 60 else random.uniform(15, 30) + lane.pause(min(delay, max(0, end - lane.clock()))) raise CapacityDeadline("no free workers available before deadline")