marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-18 04:29:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

search-with-proof: name the exhausted limit, keep partial evidence, tighter reservation

Browse Source 
budget_spent carries end_detail (spend, task_spend, reason_rounds, ...); a task
whose allowance cannot cover the next call finishes with its facts; per-task
metrics in pilot records; input reservation bytes/2 + 1024 per message (measured
3-4x over-reservation before).

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-05 20:57:56 +02:00
co-authored by Claude Fable 5.1
parent 4be2063d40
commit c33b053b15
11 changed files with 388 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openultrasast/plane/budget.py
+3 -2
View File
@@ -23,8 +23,9 @@ _USAGE_FIELDS = ("prompt_tokens", "prompt_cache_hit_tokens", "completion_tokens"
class BudgetExhausted(RuntimeError):
"""Raised before a call would start once the usd or calls ceiling is reached."""
def __init__(self, message: str, *, usd: float | None, calls: int) -> None:
super().__init__(message)
def __init__(self, message: str, *, usd: float | None, calls: int, end_detail: str = "spend") -> None:
super().__init__(f"{end_detail}: {message}")
self.end_detail = end_detail
self.usd = usd
self.calls = calls
src/openultrasast/search/board.py
+2 -2
View File
@@ -27,7 +27,7 @@ class Board:
self._lock = threading.RLock()
self.head: str | None = None
self._state: dict[str, Any] = dict(
search_id=search_id, version=0, previous=None, facts=[], intents=[], hints=[], end_reason=None, checkpoint={}
search_id=search_id, version=0, previous=None, facts=[], intents=[], hints=[], end_reason=None, end_detail=None, checkpoint={}
)
@property
@@ -54,7 +54,7 @@ class Board:
raise PermissionError("only the coordinator may write the board")
if self._state["end_reason"] is not None:
raise ValueError("search already ended")
if changes.keys() - {"facts", "intents", "hints", "end_reason", "checkpoint"}:
if changes.keys() - {"facts", "intents", "hints", "end_reason", "end_detail", "checkpoint"}:
raise ValueError("unknown board fields")
state = {**self.state, **copy.deepcopy(changes), "version": self._state["version"] + 1, "previous": self.head}
self._validate(state)
src/openultrasast/search/budget.py
+4 -2
View File
@@ -6,8 +6,10 @@ from dataclasses import dataclass
@dataclass(frozen=True)
class SearchBudget:
spend_usd: float = 0.50
max_call_usd: float = 0.05
# Recalibrated 2026-10-05 on the first three measured searches: settled cost was ~$0.003 per model call,
# but worst-case reservations (~$0.027 per call) exhausted the old $0.05 task allowance after 2-4 calls.
spend_usd: float = 2.00
max_call_usd: float = 0.40
reason_rounds: int = 4
intents_per_round: int = 3
demonstrations: int = 3
src/openultrasast/search/coordinator.py
+45 -11
View File
@@ -67,6 +67,7 @@ class Coordinator:
self.board, self.executor = board, executor
self.budget, self.clock, self.push_budget = budget or SearchBudget(), clock, push_budget
self.state = board.state
self.state.setdefault("end_detail", None)
self.progress = self.state["checkpoint"] or dict(
started_at=clock(),
spent_usd=0.0,
@@ -84,10 +85,25 @@ class Coordinator:
self.progress["wall_seconds"] = max(0, self.clock() - self.progress["started_at"])
self.state["checkpoint"] = self.progress
self.board.commit(
writer=self.board.coordinator, **{k: self.state[k] for k in ("facts", "intents", "hints", "end_reason", "checkpoint")}
writer=self.board.coordinator,
**{k: self.state[k] for k in ("facts", "intents", "hints", "end_reason", "end_detail", "checkpoint")},
)
def _end(self, reason: str) -> dict[str, Any]:
def _end(self, reason: str, end_detail: str | None = None) -> dict[str, Any]:
if reason == "budget_spent" and end_detail not in {
"spend",
"task_spend",
"reason_rounds",
"demonstrations",
"tasks",
"task_wall",
"search_wall",
"worker_steps",
"worker_wall",
"run_ceiling",
}:
raise ValueError("budget_spent requires an exhausted limit")
self.state["end_detail"] = end_detail
self.state["end_reason"] = reason
self._save()
return self.board.state
@@ -110,16 +126,21 @@ class Coordinator:
while True:
remaining = self._remaining()
if remaining <= self.budget.cleanup_seconds or self.progress["tasks"] >= self.budget.tasks:
raise BudgetExhausted("task or wall budget exhausted", usd=self.meter.spent, calls=self.progress["tasks"])
raise BudgetExhausted(
"task or wall budget exhausted",
usd=self.meter.spent,
calls=self.progress["tasks"],
end_detail="search_wall" if remaining <= self.budget.cleanup_seconds else "tasks",
)
maximum = self.budget.max_call_usd if step != "verify" else 0
reservation = self.meter.reserve(maximum)
push_reservation = None
try:
if self.push_budget is not None:
push_reservation = self.push_budget.reserve(maximum)
except BudgetExhausted:
except BudgetExhausted as exc:
reservation.release()
raise
raise BudgetExhausted("push ceiling", usd=exc.usd, calls=exc.calls, end_detail="run_ceiling") from exc
task_id = "task-" + str(self.progress["tasks"] + 1)
limits = asdict(self.budget)
limits["task_wall_seconds"] = min(self.budget.task_wall_seconds, remaining - self.budget.cleanup_seconds)
@@ -164,7 +185,16 @@ class Coordinator:
or self._remaining() <= 0
or self.clock() - dispatched_at > limits["task_wall_seconds"]
):
raise BudgetExhausted("executor exceeded cost or wall contract", usd=self.meter.spent, calls=self.progress["tasks"])
raise BudgetExhausted(
"executor exceeded cost or wall contract",
usd=self.meter.spent,
calls=self.progress["tasks"],
end_detail="task_spend"
if result.get("status") == "invalid_cost"
else "search_wall"
if self._remaining() <= 0
else "task_wall",
)
result = {**result, "task_id": task_id}
# Persist results together with their effects in run(), not a "done"
# checkpoint which could lose the result if the host then crashes.
@@ -209,13 +239,13 @@ class Coordinator:
self._save()
while True:
if self._remaining() <= self.budget.cleanup_seconds:
return self._end("budget_spent")
return self._end("budget_spent", "search_wall")
try:
intent = next((i for i in self.state["intents"] if i["status"] == "open"), None)
demo = next((f for f in self.state["facts"] if f.get("demo") and f["id"] not in self.progress["verified"]), None)
if demo is not None:
if self.progress["demonstrations"] >= self.budget.demonstrations:
return self._end("budget_spent")
return self._end("budget_spent", "demonstrations")
self.progress["demonstrations"] += 1
result = self._call("verify", {"fact_id": demo["id"], "demo": demo["demo"]})
self.progress["verified"].append(demo["id"])
@@ -259,7 +289,7 @@ class Coordinator:
intent["failure"] = "invalid_result" if result.get("status") == "ok" else result.get("status", "execution_failure")
else:
if self.progress["rounds"] >= self.budget.reason_rounds:
return self._end("budget_spent")
return self._end("budget_spent", "reason_rounds")
self.progress["rounds"] += 1
result = self._call("reason", {})
if result.get("status") == "ok" and not result.get("intents") and "complete" not in result:
@@ -275,6 +305,8 @@ class Coordinator:
if result.get("status") == "ok" and not result.get("intents") and "complete" not in result:
self.progress["failures"].append({"task_id": result["task_id"], "reason": "reason_refused"})
return self._end("reason_refused")
if result.get("budget_exhaustion"):
return self._end("budget_spent", result["budget_exhaustion"]["end_detail"])
proposals = result.get("intents", [])
if result.get("status") != "ok" or not isinstance(proposals, list) or len(proposals) > self.budget.intents_per_round:
return self._end("exhausted")
@@ -304,9 +336,11 @@ class Coordinator:
)
if not added:
return self._end("exhausted")
if result.get("budget_exhaustion"):
return self._end("budget_spent", result["budget_exhaustion"]["end_detail"])
self._save()
except BudgetExhausted:
return self._end("budget_spent")
except BudgetExhausted as exc:
return self._end("budget_spent", exc.end_detail)
@staticmethod
def _refs(refs: Any) -> bool:
src/openultrasast/search/executor.py
+28 -3
View File
@@ -274,9 +274,31 @@ class InProcessExecutor:
}
def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]:
process_limit = None
try:
uid = os.getuid()
current = 0
for entry in Path("/proc").iterdir():
if not entry.name.isdecimal():
continue
try:
status = (entry / "status").read_text()
except FileNotFoundError:
continue
fields = dict(line.split(":", 1) for line in status.splitlines() if ":" in line)
# The kernel's NPROC counter counts threads, not processes: a single IDE or agent
# process with 60 threads uses 60 of the allowance.
if int(fields["Uid"].split()[0]) == uid:
current += int(fields.get("Threads", "1").strip() or 1)
process_limit = current + 256
except (OSError, ValueError, IndexError, StopIteration):
pass
def constrain() -> None:
resource.setrlimit(resource.RLIMIT_FSIZE, (limits.get("disk_bytes", task_storage.DEFAULT_SCRATCH_BYTES),) * 2)
resource.setrlimit(resource.RLIMIT_NPROC, (128, 128))
# NPROC counts all processes of the real uid, so an absolute 128 blocks builds on busy hosts.
if process_limit is not None:
resource.setrlimit(resource.RLIMIT_NPROC, (process_limit,) * 2)
resource.setrlimit(resource.RLIMIT_AS, (limits.get("memory_bytes", 256 * 1024**2),) * 2)
resource.setrlimit(resource.RLIMIT_CPU, (math.ceil(timeout) + 1,) * 2)
@@ -292,7 +314,7 @@ class InProcessExecutor:
self.phase = "run"
timed_out = False
disk_exceeded = False
failure = None
failure: Exception | None = None
# Monitor the full disk-backed workspace, including package caches.
# An unnamespaced child can still hardcode RAM-backed paths: this
# guard is not a replacement for the operator's memory limit.
@@ -391,7 +413,10 @@ class InProcessExecutor:
if recipe["recipe"] == "maven":
argv += ["package", "-DskipTests"]
elif recipe["recipe"] == "gradle":
argv += ["assemble"]
# The project pins its Gradle version through its wrapper.
# Invoke with sh because checkout archives may lose execute bits.
wrapper = self._path(str(path.relative_to(self.repo) / "gradlew"))
argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"]
built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES})
if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
return {**built, "status": "could_not_build"}
src/openultrasast/search/worker.py
+50 -9
View File
@@ -173,6 +173,8 @@ class Worker:
self.run_budget = run_budget
self.metrics = dict(reserved=0.0, settled=0.0, model_calls=0, tokens_in=0, tokens_out=0)
self.budget_exhausted = False
self.task_calls_start = 0
self.observations: list[tuple[str, str]] = []
selected_prices = prices if prices is not None else getattr(client, "prices", None)
if not isinstance(selected_prices, Prices) or any(not math.isfinite(v) or v < 0 for v in vars(selected_prices).values()):
raise ValueError("nonnegative finite model prices required before admission")
@@ -192,26 +194,36 @@ class Worker:
def _remaining(self) -> int:
remaining = min(60, self.deadline - time.monotonic())
if remaining < 1:
raise BudgetExhausted("worker wall budget spent", usd=0, calls=0)
raise BudgetExhausted("worker wall budget spent", usd=0, calls=0, end_detail="worker_wall")
return int(remaining)
def _call(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]], meter: SpendBudget) -> dict[str, Any]:
timeout = self._remaining()
# UTF-8 bytes upper-bound content tokens; include conservative per-message
# chat framing. A provider with extra hidden billed tokens needs new pricing.
input_bound = len(json.dumps([messages, tools], ensure_ascii=False).encode()) + 4096 * (len(messages) + 1)
# Pilot measured ~27k input tokens against the old ~100k reservation (3-4x).
# Use bytes/2 plus 1024 framing tokens per message; retain max_tokens for output.
# This calibrated bound is checked at settlement, not a tokenizer guarantee.
input_bound = math.ceil(len(json.dumps([messages, tools], ensure_ascii=False).encode()) / 2) + 1024 * (len(messages) + 1)
maximum = (
input_bound * max(self.prices.input_per_m, self.prices.cache_hit_per_m) + self.max_output_tokens * self.prices.output_per_m
) / 1_000_000
reservation = meter.reserve(maximum)
try:
reservation = meter.reserve(maximum)
except BudgetExhausted as exc:
raise BudgetExhausted(
"next call reservation exceeds task allowance",
usd=exc.usd,
calls=int(self.metrics["model_calls"]) - self.task_calls_start,
end_detail="task_spend",
) from exc
run_reservation = None
try:
if self.run_budget is not None:
run_reservation = self.run_budget.reserve(maximum)
except BudgetExhausted:
except BudgetExhausted as exc:
reservation.release()
self.budget_exhausted = True
raise
raise BudgetExhausted(
"run ceiling", usd=exc.usd, calls=int(self.metrics["model_calls"]) - self.task_calls_start, end_detail="run_ceiling"
) from exc
self.metrics["reserved"] += maximum
self.metrics["model_calls"] += 1
actual = maximum
@@ -326,6 +338,7 @@ class Worker:
{"role": "user", "content": task.snapshot + "\nIntent: " + json.dumps(task.payload)},
]
self.evidence = set()
self.observations = []
for index in range(self.max_steps):
messages[0]["content"] = (
EXPLORE_PROMPT
@@ -369,13 +382,20 @@ class Worker:
ref = "tool:" + str(len(self.evidence) + 1)
self.evidence.add(ref)
result["evidence_ref"] = ref
if not result.get("error"):
text = json.dumps(result, ensure_ascii=False)
if len(text) > 2000:
text = text[:2000] + " [truncated]"
self.observations.append((ref, name + ": " + text))
except (ValueError, OSError, KeyError, TypeError) as exc:
result = {"error": type(exc).__name__, "reason": exception_reason(exc)}
messages.append({"role": "tool", "tool_call_id": call["id"], "content": json.dumps(result)})
return {"status": "execution_failure", "failure": "tool step limit exceeded"}
raise BudgetExhausted("tool step limit exceeded", usd=meter.spent, calls=self.max_steps, end_detail="worker_steps")
def __call__(self, task: SearchTask) -> dict[str, Any]:
self.budget_exhausted = False
self.task_calls_start = int(self.metrics["model_calls"])
self.observations = []
self.limits = task.limits
self.deadline = time.monotonic() + task.limits.get("task_wall_seconds", 900)
meter = SpendBudget(task.limits["max_call_usd"])
@@ -392,6 +412,27 @@ class Worker:
"cost_usd": meter.spent,
"isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"),
}
except BudgetExhausted as exc:
self.budget_exhausted = True
result = {"status": "execution_failure", "failure": exception_reason(exc)}
if task.step == "explore" and self.observations:
# Preserve literal executor observations, without inventing a security conclusion
# or making another paid call to summarize them. Bound board growth explicitly.
observations = self.observations[-6:]
result = {
"status": "ok",
"fact": {
"text": "Partial tool observations (search incomplete):\n" + "\n".join(text for _, text in observations),
"evidence_refs": [ref for ref, _ in observations],
},
}
return {
**result,
"cost_usd": meter.spent,
"end_detail": exc.end_detail,
"budget_exhaustion": {"end_detail": exc.end_detail, "usd": exc.usd, "calls": exc.calls},
"isolation_mode": getattr(self.executor, "isolation_mode", "brain-only"),
}
except _sandbox.IsolationUnavailable as exc:
return {
"status": "instrument_failure",