search-with-proof: readiness from the budget with a probe, 4096-token tool turns, 40-step tasks

HTTP start waits up to 90 s (CLI 30 s) with TCP then HTTP probing; app stdout
and stderr tails are recorded; tool-call turns get 4096 output tokens with one
retry on truncation; tasks get 40 steps with reminders.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/ax/search_probe.py

@@ -30,12 +30,12 @@ def validate_result(data: bytes, output: Path) -> None:
 
 def probe_workload(image: str, deadline: float = 900) -> Workload:
     validate_image(image)
-    return Workload(image, frozenset({"RESULT_URL"}), deadline, validate_result)
+    return Workload(image, frozenset({"RESULT_URL"}), deadline, validate_result, kind="search-probe")
 
 
 def probe_item(image: str) -> Item:
     return Item(
-        "search-probe",
+        "v1",
         {},
         "result.json",
         ("python3", "-m", "openultrasast.search.probe"),

benchmarks/ax/search_verify.py

@@ -9,6 +9,7 @@ import uuid
 from pathlib import Path
 
 from benchmarks.ax.batch import AXLane, CapacityDeadline, Item, SearchExecutorTask, Workload, validate_image
+from openultrasast.search.budget import verification_run_seconds
 from openultrasast.search.verify import RunObservation, SideRecord
 from openultrasast.search.verify_task import MAX_RESULT_BYTES, pack_checkout
 
@@ -66,7 +67,13 @@ class AXSideDispatcher:
             raise ValueError("verification accepts checkout and declarative inputs only")
         # The trusted host selects an executor for this side. Build/package work
         # finishes there before any verifier worker slot is acquired.
-        workload = Workload(self.image, frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}), timeout_seconds * 6 + 60, validate_result)
+        workload = Workload(
+            self.image,
+            frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}),
+            verification_run_seconds(demo, timeout_seconds) + 60,
+            validate_result,
+            kind="search-verify",
+        )
         lane = self.lane or AXLane(self.args, workload)
         spec = {"demo": demo, "family": family, "timeout_seconds": timeout_seconds}
         try:
@@ -79,7 +86,7 @@ class AXSideDispatcher:
             return SideRecord("could_not_build", (), 0, str(exc), isolation_mode="task-boundary")
         records = []
         for _ in range(3):
-            identity = "search-verify-" + uuid.uuid4().hex
+            identity = uuid.uuid4().hex
             item = Item(
                 identity,
                 {"VERIFY_INPUT_URL": archive},
@@ -113,4 +120,5 @@ class AXSideDispatcher:
             phase=last.phase,
             exit_code=last.exit_code,
             stderr=last.stderr,
+            stdout=last.stdout,
         )

benchmarks/search/pilot_run.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import argparse
+import copy
 import hashlib
 import io
 import json
@@ -34,7 +35,7 @@ from openultrasast.plane.memory import FileStore, open_store
 from openultrasast.provider.openrouter import OpenRouterChatClient
 from openultrasast.search import _sandbox
 from openultrasast.search.board import Board
-from openultrasast.search.budget import SearchBudget
+from openultrasast.search.budget import SearchBudget, verification_run_seconds, verification_timeout
 from openultrasast.search.coordinator import Coordinator
 from openultrasast.search.demo import FAMILY_ORACLES, load_demo, validate_oracle
 from openultrasast.search.executor import InProcessExecutor
@@ -142,6 +143,8 @@ class Pilot:
                 command_phase=diagnostic(command.phase),
                 exit_code=command.exit_code,
                 stderr=diagnostic(command.stderr, maximum=1500, private=private),
+                stdout=diagnostic(getattr(command, "stdout", ""), maximum=1500, private=private),
+                ready_seconds=getattr(command, "ready_seconds", 0),
             )
         self.record["failures"].append(entry)
 
@@ -224,26 +227,35 @@ class Pilot:
             wall = self.record["wall_seconds_by_phase"]
             wall["build_export"] = wall.get("build_export", 0) + time.monotonic() - before
 
-    def verification(self):
+    def verification(self, limits=None):
         if not FAMILY_ORACLES[self.family]:
             return VerificationRecord("no_oracle", (), 0, "no configured owned oracle")
         demo = load_demo(self.root / "demo")
         validate_oracle(self.family, demo["oracle"])
+        timeout = verification_timeout(demo, limits or self.worker.limits)
         if self.args.dry_run:
             sides = self.sides if self.args.side == "vulnerable" else (self.sides[1], self.sides[1])
-            return verify(*sides, self.root / "demo", self.family, oracle=demo["oracle"])
+            return verify(*sides, self.root / "demo", self.family, timeout_seconds=timeout, oracle=demo["oracle"])
         revisions = [self.row[self.args.side], self.row["fixed"]]
         if self.args.verify_lane == "ax":
             # Side paths are opaque lookup keys; no checkout or opposite revision reaches the brain.
             sides = [Side(self.root / str(i)) for i in range(2)]
             pins = {s.checkout: pin for s, pin in zip(sides, revisions, strict=True)}
             parent = self
+            verify_lane = copy.copy(self.lane)
+            verify_lane.workload = Workload(
+                self.args.image,
+                frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}),
+                verification_run_seconds(demo, timeout) + 60,
+                validate_result,
+                kind="search-verify",
+            )
 
             class CountingLane:
                 def __call__(self, *a, **kw):
                     parent.admit_task()
                     parent.record["verify_tasks"] += 1
-                    return parent.lane(*a, **kw)
+                    return verify_lane(*a, **kw)
 
             dispatcher = AXSideDispatcher(
                 self.args,
@@ -256,6 +268,7 @@ class Pilot:
                 *sides,
                 self.root / "demo",
                 self.family,
+                timeout_seconds=timeout,
                 oracle=demo["oracle"],
                 task_dispatcher=dispatcher,
                 browser=BrowserExecutor(task_boundary=True) if demo["oracle"] == "xss" else None,
@@ -263,7 +276,7 @@ class Pilot:
         if _sandbox.isolation_mode() != "userns":
             raise _sandbox.IsolationUnavailable("VM verification requires userns")
         sides = []
-        spec = dict(demo=demo, family=self.family, timeout_seconds=5)
+        spec = dict(demo=demo, family=self.family, timeout_seconds=timeout)
         for revision in revisions:
             archive = self.prepare(revision, spec)
             bundle = self.root / ("built-" + uuid.uuid4().hex)
@@ -280,6 +293,7 @@ class Pilot:
             *sides,
             self.root / "verify-demo",
             self.family,
+            timeout_seconds=timeout,
             oracle=demo["oracle"],
             browser=BrowserExecutor() if demo["oracle"] == "xss" else None,
         )
@@ -293,7 +307,7 @@ class Pilot:
         end = "execution_failure"
         try:
             if task.step == "verify":
-                result = self.verification()
+                result = self.verification(task.limits)
                 self.verifications.append(result.outcome)
                 self.record["outcome"] = result.outcome
                 if self.args.side == "fixed" and result.sides:
@@ -393,7 +407,7 @@ def main(argv=None):
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--verify-lane", choices=("vm", "ax"), default="vm")
     parser.add_argument("--model", default=DEFAULT_DETECTOR_MODEL)
-    parser.add_argument("--max-tokens", type=int, default=1024)
+    parser.add_argument("--max-tokens", type=int, default=4096)
     parser.add_argument("--usd-per-mtok-in", type=float)
     parser.add_argument("--usd-per-mtok-out", type=float)
     parser.add_argument("--image")
@@ -501,7 +515,9 @@ def main(argv=None):
         try:
             store = open_store(args.board_memory) if args.board_memory else FileStore(root / "boards")
             if not args.dry_run:
-                lane = AXLane(args, Workload(args.image, frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}), 900, validate_result))
+                lane = AXLane(
+                    args, Workload(args.image, frozenset({"VERIFY_INPUT_URL", "RESULT_URL"}), 900, validate_result, kind="search-verify")
+                )
         except Exception as exc:
             aborted_reason = exception_reason(exc)
         for index in indices if aborted_reason is None else []: