Merge branch 'main' into search-with-proof

# Conflicts:
#	tests/test_search_task_storage.py
This commit is contained in:
Marius Ciepluch
2026-10-05 20:58:10 +02:00
14 changed files with 599 additions and 148 deletions
+8 -3
View File
@@ -71,7 +71,8 @@ def apply_when_available(lane, document, end):
if lane.clock() >= next_health_check:
lane.check_cluster_health(timeout=min(10, max(0.01, end - lane.clock())))
next_health_check = lane.clock() + 60
# Brief snapshot-copy contention clears quickly; busy workers need slower retries.
# Substrate's golden-snapshot copy holds the other worker for about a second after each new task
# (operator, 2026-10-05): retry quickly for the first minute, then back off while real tasks run.
delay = random.uniform(2, 5) if lane.clock() - started < 60 else random.uniform(15, 30)
lane.pause(min(delay, max(0, end - lane.clock())))
raise CapacityDeadline("no free workers available before deadline")
@@ -261,8 +262,10 @@ class AXLane:
def _execute_attempt(self, item, output, attempt, attempts):
pin_id = item.id
digest = hashlib.sha256(f"{self.run}/{pin_id}/attempt-{attempt}".encode()).hexdigest()[:12]
slug = re.sub(r"[^a-z0-9]+", "-", str(pin_id).lower()).strip("-")[:22]
name = f"ousast-engine-{slug or 'pin'}-{digest}"
kind = re.sub(r"[^a-z0-9]+", "-", self.workload.kind.lower()).strip("-") or "pin"
# Names are capped at 49 bytes: 14 (prefix) + kind + 1 + slug + 1 + 12 (digest).
slug = re.sub(r"[^a-z0-9]+", "-", str(pin_id).lower()).strip("-")[: max(4, 21 - len(kind))]
name = f"ousast-engine-{kind}-{slug or 'pin'}-{digest}"
key_id = hashlib.sha256(str(pin_id).encode()).hexdigest()
prefix = f"engine-queue/{self.run}/{key_id}/attempt-{attempt}/"
keys = [prefix + item.object_names.get(key, key.lower() + PRESIGNED_ARCHIVE_SUFFIX) for key in item.inputs]
@@ -360,6 +363,8 @@ class Workload:
url_env: frozenset[str]
deadline: float
validate: object
# Task-name prefix after "ousast-engine-"; the operator's per-host egress rules match on it.
kind: str = "pin"
def validate_image(image):
+1 -1
View File
@@ -27,7 +27,7 @@ def task_manifest(name, args, urls):
def engine_workload(args, validate):
return Workload(args.image, frozenset({"SOURCE_URL", "QUESTIONS_URL", "RESULT_URL"}), args.deadline, validate)
return Workload(args.image, frozenset({"SOURCE_URL", "QUESTIONS_URL", "RESULT_URL"}), args.deadline, validate, kind="pass")
def engine_item(pin, pin_id, source, args):
+1 -1
View File
@@ -220,7 +220,7 @@ def run_replays(changes, args, out, lane, *, deadline_scale=1.0, deadline=None):
maximum = max(
max(json.loads(i.inputs["SPEC_URL"])["deadline_seconds"], json.loads(i.inputs["SPEC_URL"])["base_deadline_seconds"]) for i in items
)
workload = Workload(args.image, frozenset({"SPEC_URL", "RESULT_URL"}), maximum + 600, validate_result)
workload = Workload(args.image, frozenset({"SPEC_URL", "RESULT_URL"}), maximum + 600, validate_result, kind="replay")
lanes = ("ax", "ax", "docker") if lane == "mixed" else (lane,) * (2 if lane in {"ax", "kind"} else 1)
original = {item.id: change for item, change in zip(items, changes, strict=True)}
executors = {}