mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 20:19:29 +00:00
search-with-proof group 4 + image: board, spend reservation, coordinator, search-task image
Board as immutable versioned blobs (coordinator sole writer); budget reserve/settle before model calls; per-task and run caps; coordinator loop over an injected executor; batch.py accepts a fixed allow-list of search variables; XSS browser oracle; ousast-search-task image (Python, Node/TypeScript, PHP, Java 17, sqlite, bwrap, chromium) built by CI. 103 tests on the host. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
6394ee473d
commit
4b38239f68
@@ -83,6 +83,9 @@ SPEC_OWNED = {
|
||||
|
||||
# Reachable modules that stand alone: their own CLI subcommand or external protocol, off the scan path.
|
||||
STANDALONE = {
|
||||
"search.coordinator": (
|
||||
"search-with-proof Req 1/2/4: injected search executor and offline board checkpoint inspector; agent worker pending"
|
||||
),
|
||||
"push_scoring": "pre-push-safety-net task 1.3: frozen-profile diagnostic outcome scorer; python -m openultrasast.push_scoring",
|
||||
"push_inputs": "pre-push-safety-net task 1.2: offline input provenance validator; python -m openultrasast.push_inputs",
|
||||
"mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration",
|
||||
|
||||
@@ -10,6 +10,7 @@ failure is `AccountError` (the task is `failed` and the run starts nothing furth
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from decimal import Decimal
|
||||
from typing import Any, Protocol
|
||||
|
||||
from ..model.endpoint import Prices
|
||||
@@ -251,5 +252,70 @@ class MeteredEmbeddingClient:
|
||||
|
||||
__all__ = [
|
||||
"AccountError", "BudgetExhausted", "EmbeddingClient", "MeteredClient", "MeteredEmbeddingClient", "UnmeteredUsage", "cost_of",
|
||||
"is_account_error", "prices_from",
|
||||
"is_account_error", "prices_from", "SpendBudget", "SpendReservation",
|
||||
] # fmt: skip
|
||||
|
||||
|
||||
class SpendBudget:
|
||||
"""Thread-safe pre-call admission. Share one instance across concurrent callers.
|
||||
|
||||
A caller must supply a genuine upper bound, including all tool-loop calls.
|
||||
Reservations remain charged until settled or explicitly released on failure.
|
||||
Existing MeteredClient callers retain their historical post-call metering.
|
||||
"""
|
||||
|
||||
def __init__(self, cap: float, *, spent: float = 0.0) -> None:
|
||||
from threading import Lock
|
||||
|
||||
self._lock = Lock()
|
||||
self._cap = self._amount(cap)
|
||||
self._spent = self._amount(spent)
|
||||
if self._spent > self._cap:
|
||||
raise ValueError("spend exceeds cap")
|
||||
self._reserved = Decimal(0)
|
||||
|
||||
@staticmethod
|
||||
def _amount(value: float) -> Decimal:
|
||||
|
||||
amount = Decimal(str(value))
|
||||
if not amount.is_finite() or amount < 0:
|
||||
raise ValueError("cost must be finite and nonnegative")
|
||||
return amount
|
||||
|
||||
@property
|
||||
def spent(self) -> float:
|
||||
with self._lock:
|
||||
return float(self._spent)
|
||||
|
||||
@property
|
||||
def reserved(self) -> float:
|
||||
with self._lock:
|
||||
return float(self._reserved)
|
||||
|
||||
def reserve(self, max_cost: float) -> SpendReservation:
|
||||
amount = self._amount(max_cost)
|
||||
with self._lock:
|
||||
if self._spent + self._reserved + amount > self._cap:
|
||||
raise BudgetExhausted("spend reservation exceeds cap", usd=float(self._spent), calls=0)
|
||||
self._reserved += amount
|
||||
return SpendReservation(self, amount)
|
||||
|
||||
|
||||
class SpendReservation:
|
||||
def __init__(self, budget: SpendBudget, amount: Decimal) -> None:
|
||||
self._budget, self._amount, self._active = budget, amount, True
|
||||
|
||||
def settle(self, actual: float) -> None:
|
||||
amount = self._budget._amount(actual)
|
||||
with self._budget._lock:
|
||||
if not self._active:
|
||||
raise ValueError("reservation already closed")
|
||||
if amount > self._amount:
|
||||
# Retain the reservation: a broken upper bound must fail closed.
|
||||
raise ValueError("actual cost exceeds reservation")
|
||||
self._budget._reserved -= self._amount
|
||||
self._budget._spent += amount
|
||||
self._active = False
|
||||
|
||||
def release(self) -> None:
|
||||
self.settle(0)
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Coordinator-owned, content-addressed search checkpoints in the plane memory store.
|
||||
|
||||
Workers receive snapshots and return proposals; they never receive this writer or
|
||||
store credentials. Persist the returned head in the host's run checkpoint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from ..plane.memory import MemoryStore
|
||||
|
||||
|
||||
class Board:
|
||||
def __init__(self, store: MemoryStore, search_id: str, *, coordinator: str, max_bytes: int = 262144) -> None:
|
||||
if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,63}", search_id) or not coordinator or max_bytes < 256:
|
||||
raise ValueError("invalid board configuration")
|
||||
self.store, self.coordinator, self.max_bytes = store, coordinator, max_bytes
|
||||
self.prefix = "boards/" + hashlib.sha256(search_id.encode()).hexdigest()
|
||||
self._lock = threading.RLock()
|
||||
self.head: str | None = None
|
||||
self._state: dict[str, Any] = dict(
|
||||
search_id=search_id, version=0, previous=None, facts=[], intents=[], hints=[], end_reason=None, checkpoint={}
|
||||
)
|
||||
|
||||
@property
|
||||
def state(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
return copy.deepcopy(self._state)
|
||||
|
||||
@classmethod
|
||||
def resume(cls, store: MemoryStore, search_id: str, head: str, *, coordinator: str, max_bytes: int = 262144) -> Board:
|
||||
board = cls(store, search_id, coordinator=coordinator, max_bytes=max_bytes)
|
||||
data = store.get_blob(board.prefix, head)
|
||||
if data is None or len(data) > max_bytes:
|
||||
raise ValueError("missing or oversized board checkpoint")
|
||||
state = json.loads(data)
|
||||
if state["search_id"] != search_id:
|
||||
raise ValueError("checkpoint belongs to another search")
|
||||
board._validate(state)
|
||||
board._state, board.head = state, head
|
||||
return board
|
||||
|
||||
def commit(self, *, writer: str, **changes: Any) -> str:
|
||||
with self._lock:
|
||||
if writer != self.coordinator:
|
||||
raise PermissionError("only the coordinator may write the board")
|
||||
if self._state["end_reason"] is not None:
|
||||
raise ValueError("search already ended")
|
||||
if changes.keys() - {"facts", "intents", "hints", "end_reason", "checkpoint"}:
|
||||
raise ValueError("unknown board fields")
|
||||
state = {**self.state, **copy.deepcopy(changes), "version": self._state["version"] + 1, "previous": self.head}
|
||||
self._validate(state)
|
||||
data = json.dumps(state, sort_keys=True, allow_nan=False).encode()
|
||||
if len(data) > self.max_bytes:
|
||||
raise ValueError("board size cap exceeded")
|
||||
head = self.store.put_blob(self.prefix, data)
|
||||
self._state, self.head = state, head
|
||||
return head
|
||||
|
||||
@staticmethod
|
||||
def _validate(state: dict[str, Any]) -> None:
|
||||
if state["end_reason"] not in (None, "goal_met", "budget_spent", "exhausted"):
|
||||
raise ValueError("invalid end reason")
|
||||
ids: dict[str, set[str]] = {}
|
||||
for kind in ("facts", "intents", "hints"):
|
||||
ids[kind] = set()
|
||||
for row in state[kind]:
|
||||
required = ("id", "step", "task_id", "description" if kind == "intents" else "text")
|
||||
if any(not isinstance(row.get(key), str) or not row[key] for key in required):
|
||||
raise ValueError("entry requires text and writer provenance")
|
||||
if row["id"] in ids[kind]:
|
||||
raise ValueError("duplicate board id")
|
||||
ids[kind].add(row["id"])
|
||||
for fact in state["facts"]:
|
||||
if not isinstance(fact.get("evidence_refs"), list) or not fact["evidence_refs"]:
|
||||
raise ValueError("facts require evidence references")
|
||||
if any(not isinstance(ref, str) or not ref for ref in fact["evidence_refs"]):
|
||||
raise ValueError("invalid evidence reference")
|
||||
if not isinstance(fact.get("from_intents"), list) or set(fact["from_intents"]) - ids["intents"]:
|
||||
raise ValueError("unknown source intent")
|
||||
for intent in state["intents"]:
|
||||
if not isinstance(intent.get("from_facts"), list) or set(intent["from_facts"]) - ids["facts"]:
|
||||
raise ValueError("unknown source fact")
|
||||
if intent.get("status") not in ("open", "claimed", "concluded"):
|
||||
raise ValueError("invalid intent status")
|
||||
if intent["status"] == "claimed" and not intent.get("claimant"):
|
||||
raise ValueError("claimed intent requires claimant")
|
||||
if intent["status"] == "open" and intent.get("claimant") is not None:
|
||||
raise ValueError("open intent cannot have claimant")
|
||||
|
||||
def snapshot(self, *, max_bytes: int = 32768) -> str:
|
||||
"""Valid YAML even when truncated; explicit omission counts, never a cut string."""
|
||||
if max_bytes < 128:
|
||||
raise ValueError("snapshot cap must be at least 128 bytes")
|
||||
state = self.state
|
||||
view = {key: state[key] for key in ("version", "end_reason", "facts", "intents", "hints")}
|
||||
view["omitted"] = dict.fromkeys(("facts", "intents", "hints"), 0)
|
||||
while True:
|
||||
text = str(yaml.safe_dump(view, sort_keys=False, allow_unicode=True))
|
||||
if len(text.encode()) <= max_bytes:
|
||||
return text
|
||||
kind = next((k for k in ("facts", "intents", "hints") if view[k]), None)
|
||||
if kind is None:
|
||||
raise ValueError("snapshot cap too small")
|
||||
view[kind].pop(0)
|
||||
view["omitted"][kind] += 1
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Search ceilings carried to every executor; no model or transport dependencies."""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SearchBudget:
|
||||
spend_usd: float = 0.50
|
||||
max_call_usd: float = 0.05
|
||||
reason_rounds: int = 4
|
||||
intents_per_round: int = 3
|
||||
demonstrations: int = 3
|
||||
tasks: int = 32
|
||||
retries: int = 1
|
||||
memory_bytes: int = 5 * 1024**3
|
||||
disk_bytes: int = 2 * 1024**3
|
||||
task_wall_seconds: float = 900
|
||||
wall_seconds: float = 4 * 3600
|
||||
cleanup_seconds: float = 30
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name, value in vars(self).items():
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
||||
raise ValueError(f"invalid {name}")
|
||||
if value < 0 or (value == 0 and name not in ("retries", "spend_usd")):
|
||||
raise ValueError(f"invalid {name}")
|
||||
if name in (
|
||||
"reason_rounds",
|
||||
"intents_per_round",
|
||||
"demonstrations",
|
||||
"tasks",
|
||||
"retries",
|
||||
"memory_bytes",
|
||||
"disk_bytes",
|
||||
) and not isinstance(value, int):
|
||||
raise ValueError(f"{name} must be an integer")
|
||||
@@ -0,0 +1,348 @@
|
||||
"""Host-only board driver. Executors own task isolation and enforce supplied limits.
|
||||
|
||||
Reason/explore workers are group 5. They return proposals, never board writes.
|
||||
Only the trusted verify executor may return a demonstrated outcome. An interrupted
|
||||
call is conservatively charged its reservation on resume, never silently free.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from ..plane.budget import BudgetExhausted, SpendBudget
|
||||
from ..plane.memory import open_store
|
||||
from .board import Board
|
||||
from .budget import SearchBudget
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SearchTask:
|
||||
id: str
|
||||
search_id: str
|
||||
step: str
|
||||
snapshot: str
|
||||
payload: dict[str, Any]
|
||||
limits: dict[str, Any]
|
||||
lane: str = "ax"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for value in (self.id, self.search_id):
|
||||
if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,63}", value):
|
||||
raise ValueError("invalid search task identifier")
|
||||
if self.step not in ("reason", "explore", "verify") or self.lane not in ("ax", "kind", "docker"):
|
||||
raise ValueError("invalid search task step or lane")
|
||||
|
||||
|
||||
class Executor(Protocol):
|
||||
def __call__(self, task: SearchTask) -> dict[str, Any]:
|
||||
"""Finish/clean up within limits; report actual cost, including failed calls.
|
||||
|
||||
A missing cost on a model task charges its entire reservation. Model
|
||||
workers must reserve each call against limits.max_call_usd (the task's
|
||||
total allowance) using plane.budget.SpendBudget. Exceptions are untrusted
|
||||
diagnostics and must never be copied into the board.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class Coordinator:
|
||||
def __init__(
|
||||
self,
|
||||
board: Board,
|
||||
executor: Executor,
|
||||
*,
|
||||
budget: SearchBudget | None = None,
|
||||
clock: Callable[[], float] = time.time,
|
||||
push_budget: SpendBudget | None = None,
|
||||
) -> None:
|
||||
self.board, self.executor = board, executor
|
||||
self.budget, self.clock, self.push_budget = budget or SearchBudget(), clock, push_budget
|
||||
self.state = board.state
|
||||
self.progress = self.state["checkpoint"] or dict(
|
||||
started_at=clock(),
|
||||
spent_usd=0.0,
|
||||
tasks=0,
|
||||
rounds=0,
|
||||
retries=0,
|
||||
demonstrations=0,
|
||||
pending=None,
|
||||
verified=[],
|
||||
failures=[],
|
||||
)
|
||||
self.meter = SpendBudget(self.budget.spend_usd, spent=self.progress["spent_usd"])
|
||||
|
||||
def _save(self) -> None:
|
||||
self.progress["wall_seconds"] = max(0, self.clock() - self.progress["started_at"])
|
||||
self.state["checkpoint"] = self.progress
|
||||
self.board.commit(
|
||||
writer=self.board.coordinator, **{k: self.state[k] for k in ("facts", "intents", "hints", "end_reason", "checkpoint")}
|
||||
)
|
||||
|
||||
def _end(self, reason: str) -> dict[str, Any]:
|
||||
self.state["end_reason"] = reason
|
||||
self._save()
|
||||
return self.board.state
|
||||
|
||||
def _remaining(self) -> float:
|
||||
return float(self.budget.wall_seconds - max(0, self.clock() - self.progress["started_at"]))
|
||||
|
||||
def _call(self, step: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
lane = "ax"
|
||||
while True:
|
||||
remaining = self._remaining()
|
||||
if remaining <= self.budget.cleanup_seconds or self.progress["tasks"] >= self.budget.tasks:
|
||||
raise BudgetExhausted("task or wall budget exhausted", usd=self.meter.spent, calls=self.progress["tasks"])
|
||||
maximum = self.budget.max_call_usd if step != "verify" else 0
|
||||
reservation = self.meter.reserve(maximum)
|
||||
push_reservation = None
|
||||
try:
|
||||
if self.push_budget is not None:
|
||||
push_reservation = self.push_budget.reserve(maximum)
|
||||
except BudgetExhausted:
|
||||
reservation.release()
|
||||
raise
|
||||
task_id = "task-" + str(self.progress["tasks"] + 1)
|
||||
limits = asdict(self.budget)
|
||||
limits["task_wall_seconds"] = min(self.budget.task_wall_seconds, remaining - self.budget.cleanup_seconds)
|
||||
task = SearchTask(task_id, self.state["search_id"], step, self.board.snapshot(), payload, limits, lane)
|
||||
self.progress["tasks"] += 1
|
||||
self.progress["pending"] = {"id": task_id, "step": step, "payload": payload}
|
||||
# A crash after this durable write may have spent the whole allowance.
|
||||
self.progress["spent_usd"] = self.meter.spent + maximum
|
||||
if step == "explore":
|
||||
intent = next(i for i in self.state["intents"] if i["id"] == payload["intent_id"])
|
||||
intent.update(status="claimed", claimant=task_id)
|
||||
self._save()
|
||||
task = replace(task, snapshot=self.board.snapshot())
|
||||
dispatched_at = self.clock()
|
||||
try:
|
||||
result = self.executor(task)
|
||||
except Exception:
|
||||
# Includes cleanup failures: never launch more work after a task
|
||||
# may have leaked. The persisted reservation remains conservative.
|
||||
reservation.settle(maximum)
|
||||
if push_reservation is not None:
|
||||
push_reservation.settle(maximum)
|
||||
raise
|
||||
if not isinstance(result, dict):
|
||||
result = {"status": "invalid_result"}
|
||||
actual = result.get("cost_usd", maximum)
|
||||
try:
|
||||
reservation.settle(actual)
|
||||
if push_reservation is not None:
|
||||
push_reservation.settle(actual)
|
||||
except (ValueError, TypeError, ArithmeticError):
|
||||
# Charge the maximum and stop on a violated executor contract.
|
||||
if reservation._active:
|
||||
reservation.settle(maximum)
|
||||
if push_reservation is not None and push_reservation._active:
|
||||
push_reservation.settle(maximum)
|
||||
result = {"status": "invalid_cost"}
|
||||
self.progress["spent_usd"] = self.meter.spent
|
||||
self.progress["pending"] = None
|
||||
if (
|
||||
result.get("status") == "invalid_cost"
|
||||
or self._remaining() <= 0
|
||||
or self.clock() - dispatched_at > limits["task_wall_seconds"]
|
||||
):
|
||||
raise BudgetExhausted("executor exceeded cost or wall contract", usd=self.meter.spent, calls=self.progress["tasks"])
|
||||
result = {**result, "task_id": task_id}
|
||||
# Persist results together with their effects in run(), not a "done"
|
||||
# checkpoint which could lose the result if the host then crashes.
|
||||
if result.get("status") in ("sandbox_failure", "oom", "instrument_failure") and self.progress["retries"] < self.budget.retries:
|
||||
self.progress["retries"] += 1
|
||||
lane = "docker"
|
||||
continue
|
||||
return result
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
if self.state["end_reason"]:
|
||||
return self.board.state
|
||||
pending = self.progress.get("pending")
|
||||
if pending:
|
||||
# Do not duplicate a possibly paid operation. Recover the claim as a
|
||||
# recorded failure; the next reason task can propose another direction.
|
||||
self.progress["failures"].append({"task_id": pending["id"], "reason": "interrupted"})
|
||||
for intent in self.state["intents"]:
|
||||
if intent.get("claimant") == pending["id"]:
|
||||
intent.update(status="concluded", failure="interrupted")
|
||||
if pending["step"] == "verify":
|
||||
self.progress["verified"].append(pending["payload"]["fact_id"])
|
||||
self.progress["pending"] = None
|
||||
self._save()
|
||||
while True:
|
||||
if self._remaining() <= self.budget.cleanup_seconds:
|
||||
return self._end("budget_spent")
|
||||
try:
|
||||
intent = next((i for i in self.state["intents"] if i["status"] == "open"), None)
|
||||
demo = next((f for f in self.state["facts"] if f.get("demo") and f["id"] not in self.progress["verified"]), None)
|
||||
if demo is not None:
|
||||
if self.progress["demonstrations"] >= self.budget.demonstrations:
|
||||
return self._end("budget_spent")
|
||||
self.progress["demonstrations"] += 1
|
||||
result = self._call("verify", {"fact_id": demo["id"], "demo": demo["demo"]})
|
||||
self.progress["verified"].append(demo["id"])
|
||||
refs = result.get("evidence_refs")
|
||||
if result.get("status") == "ok" and result.get("outcome") == "demonstrated" and self._refs(refs):
|
||||
self.state["facts"].append(
|
||||
dict(
|
||||
id="verified-" + result["task_id"],
|
||||
text="Differential demonstrated",
|
||||
evidence_refs=refs,
|
||||
from_intents=demo["from_intents"],
|
||||
step="verify",
|
||||
task_id=result["task_id"],
|
||||
)
|
||||
)
|
||||
return self._end("goal_met")
|
||||
self.progress["failures"].append({"task_id": result["task_id"], "reason": "not_demonstrated"})
|
||||
elif intent is not None:
|
||||
result = self._call("explore", {"intent_id": intent["id"], "description": intent["description"]})
|
||||
intent["status"] = "concluded"
|
||||
fact = result.get("fact")
|
||||
if (
|
||||
result.get("status") == "ok"
|
||||
and isinstance(fact, dict)
|
||||
and isinstance(fact.get("text"), str)
|
||||
and fact["text"]
|
||||
and self._refs(fact.get("evidence_refs"))
|
||||
):
|
||||
entry = dict(
|
||||
id="fact-" + result["task_id"],
|
||||
text=fact["text"],
|
||||
evidence_refs=fact["evidence_refs"],
|
||||
from_intents=[intent["id"]],
|
||||
step="explore",
|
||||
task_id=result["task_id"],
|
||||
)
|
||||
if isinstance(fact.get("demo"), str) and fact["demo"]:
|
||||
entry["demo"] = fact["demo"]
|
||||
self.state["facts"].append(entry)
|
||||
else:
|
||||
intent["failure"] = "invalid_result" if result.get("status") == "ok" else result.get("status", "execution_failure")
|
||||
else:
|
||||
if self.progress["rounds"] >= self.budget.reason_rounds:
|
||||
return self._end("budget_spent")
|
||||
self.progress["rounds"] += 1
|
||||
result = self._call("reason", {})
|
||||
proposals = result.get("intents", [])
|
||||
if result.get("status") != "ok" or not isinstance(proposals, list) or len(proposals) > self.budget.intents_per_round:
|
||||
return self._end("exhausted")
|
||||
known = {f["id"] for f in self.state["facts"]}
|
||||
descriptions = {i["description"] for i in self.state["intents"]}
|
||||
added = 0
|
||||
for proposal in proposals:
|
||||
if not isinstance(proposal, dict):
|
||||
continue
|
||||
description, refs = proposal.get("description"), proposal.get("from_facts", [])
|
||||
if not isinstance(description, str) or not description.strip() or description in descriptions:
|
||||
continue
|
||||
if not isinstance(refs, list) or any(not isinstance(ref, str) or ref not in known for ref in refs):
|
||||
continue
|
||||
added += 1
|
||||
descriptions.add(description)
|
||||
self.state["intents"].append(
|
||||
dict(
|
||||
id=result["task_id"] + "-intent-" + str(added),
|
||||
description=description,
|
||||
from_facts=refs,
|
||||
status="open",
|
||||
claimant=None,
|
||||
step="reason",
|
||||
task_id=result["task_id"],
|
||||
)
|
||||
)
|
||||
if not added:
|
||||
return self._end("exhausted")
|
||||
self._save()
|
||||
except BudgetExhausted:
|
||||
return self._end("budget_spent")
|
||||
|
||||
@staticmethod
|
||||
def _refs(refs: Any) -> bool:
|
||||
return isinstance(refs, list) and bool(refs) and all(isinstance(ref, str) and bool(ref) for ref in refs)
|
||||
|
||||
|
||||
class BatchExecutor:
|
||||
"""Adapter to the existing AX/VM transfer and checkpoint protocol.
|
||||
|
||||
Pass AXLane/DockerLane instances and a trusted worker command. No default
|
||||
worker is invented here: reason/explore entrypoints belong to group 5.
|
||||
The coordinator owns retries, so batch performs exactly one attempt.
|
||||
"""
|
||||
|
||||
def __init__(self, *, image: str, command: tuple[str, ...], output: Path, lanes: dict[str, Any]) -> None:
|
||||
batch = import_module("benchmarks.ax.batch")
|
||||
batch.validate_image(image)
|
||||
if not command:
|
||||
raise ValueError("worker command required")
|
||||
self.image, self.command, self.output, self.lanes = image, command, output, lanes
|
||||
|
||||
def __call__(self, task: SearchTask) -> dict[str, Any]:
|
||||
import copy
|
||||
|
||||
batch = import_module("benchmarks.ax.batch")
|
||||
|
||||
if task.lane not in self.lanes:
|
||||
return {"status": "sandbox_failure", "cost_usd": 0}
|
||||
data = json.dumps(asdict(task), sort_keys=True, allow_nan=False).encode()
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
identity = task.search_id + "-" + task.id
|
||||
item = batch.Item(
|
||||
identity,
|
||||
{"SPEC_URL": data},
|
||||
"result.dat",
|
||||
self.command,
|
||||
{"SEARCH_STEP": task.step, "SEARCH_ID": task.search_id, "SEARCH_TASK_ID": task.id},
|
||||
digest,
|
||||
)
|
||||
|
||||
def validate(raw: bytes, output: Path) -> None:
|
||||
if len(raw) > 262144:
|
||||
raise ValueError("search result size exceeded")
|
||||
record = json.loads(raw)
|
||||
if not isinstance(record, dict) or record.get("status") not in (
|
||||
"ok",
|
||||
"sandbox_failure",
|
||||
"instrument_failure",
|
||||
"oom",
|
||||
"execution_failure",
|
||||
):
|
||||
raise ValueError("invalid search result")
|
||||
(output / "result.json").write_text(json.dumps(record))
|
||||
|
||||
workload = batch.Workload(self.image, frozenset({"SPEC_URL", "RESULT_URL"}), task.limits["task_wall_seconds"], validate)
|
||||
lane = copy.copy(self.lanes[task.lane])
|
||||
if hasattr(lane, "workload"):
|
||||
lane.workload = workload
|
||||
out = self.output / task.search_id / task.id
|
||||
batch.dispatch([item], workload, out, {task.lane: lane}, lanes=(task.lane,), max_attempts=1)
|
||||
record_path = out / (hashlib.sha256(identity.encode()).hexdigest() + ".json")
|
||||
if not record_path.exists():
|
||||
return {"status": "execution_failure", "cost_usd": 0}
|
||||
record: dict[str, Any] = json.loads(record_path.read_text())
|
||||
return record
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Offline checkpoint inspection, also useful before resuming with an executor."""
|
||||
parser = argparse.ArgumentParser(description="Inspect a search board checkpoint (no model calls)")
|
||||
parser.add_argument("--memory", required=True)
|
||||
parser.add_argument("--search-id", required=True)
|
||||
parser.add_argument("--head", required=True)
|
||||
args = parser.parse_args()
|
||||
board = Board.resume(open_store(args.memory), args.search_id, args.head, coordinator="inspect")
|
||||
print(board.snapshot(), end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -11,12 +11,16 @@ import json
|
||||
import os
|
||||
import secrets
|
||||
import selectors
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -136,18 +140,111 @@ class SSRFOracle(OutputOracle):
|
||||
self.process.stdout.close()
|
||||
|
||||
|
||||
BrowserExecutor = Callable[[str, str], bool]
|
||||
class BrowserExecutor:
|
||||
"""Run captured HTML with a verifier-owned alert canary.
|
||||
|
||||
Payloads call alert("ousast-xss"). Reflection/escaped HTML cannot set the fresh
|
||||
DOM marker. A restrictive CSP disables external resources and connections;
|
||||
Chromium's sandbox stays enabled and each capture uses a fresh profile.
|
||||
Launch failures are unavailable execution, never a negative observation.
|
||||
"""
|
||||
|
||||
def __init__(self, binary: str | None = None, *, timeout_seconds: float = 15) -> None:
|
||||
selected = binary or shutil.which("chromium") or shutil.which("chromium-browser")
|
||||
self.timeout_seconds = timeout_seconds
|
||||
if not selected:
|
||||
raise OSError("headless chromium unavailable")
|
||||
self.binary: str = selected
|
||||
|
||||
def __call__(self, document: str, nonce: str) -> bool:
|
||||
marker = secrets.token_hex(24)
|
||||
# Neither the marker nor its attribute appears in app output. The bootstrap
|
||||
# source does appear in dump-dom, so inspect an actual DOM attribute only.
|
||||
attribute = "data-proof-" + secrets.token_hex(12)
|
||||
prelude = (
|
||||
'<meta http-equiv="Content-Security-Policy" content="default-src \'none\'; '
|
||||
"script-src 'unsafe-inline'; base-uri 'none'; form-action 'none'\">"
|
||||
'<script>window.alert=function(value){if(value==="ousast-xss"||value==='
|
||||
+ json.dumps(nonce)
|
||||
+ "){document.documentElement.setAttribute("
|
||||
+ json.dumps(attribute)
|
||||
+ ","
|
||||
+ json.dumps(marker)
|
||||
+ ");}};</script>"
|
||||
)
|
||||
with tempfile.TemporaryDirectory(prefix="ousast-browser-") as profile:
|
||||
# HTML can navigate even with CSP. A private network namespace, not
|
||||
# browser flags, is the no-egress boundary. Expose runtime only.
|
||||
sandbox = ["bwrap", "--unshare-all", "--die-with-parent", "--new-session"]
|
||||
for path in ("/usr", "/bin", "/lib", "/lib64"):
|
||||
if Path(path).exists():
|
||||
sandbox += ["--ro-bind", path, path]
|
||||
sandbox += [
|
||||
"--dev",
|
||||
"/dev",
|
||||
"--proc",
|
||||
"/proc",
|
||||
"--tmpfs",
|
||||
"/tmp",
|
||||
"--bind",
|
||||
profile,
|
||||
"/profile",
|
||||
"--setenv",
|
||||
"HOME",
|
||||
"/profile",
|
||||
"--",
|
||||
]
|
||||
try:
|
||||
done = subprocess.run(
|
||||
[
|
||||
*sandbox,
|
||||
self.binary,
|
||||
"--headless",
|
||||
"--dump-dom",
|
||||
"--disable-gpu",
|
||||
"--disable-background-networking",
|
||||
"--disable-extensions",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--disable-sync",
|
||||
"--user-data-dir=/profile",
|
||||
"--virtual-time-budget=1000",
|
||||
"data:text/html;charset=utf-8," + quote(prelude + document),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self.timeout_seconds,
|
||||
env={"PATH": "/usr/bin:/bin", "HOME": profile},
|
||||
)
|
||||
except subprocess.SubprocessError as exc:
|
||||
raise OSError("headless chromium failed") from exc
|
||||
if done.returncode or "<html" not in done.stdout.lower():
|
||||
raise OSError("headless chromium did not produce a document: " + done.stderr[-500:])
|
||||
|
||||
class Marker(HTMLParser):
|
||||
seen = False
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag == "html" and (attribute, marker) in attrs:
|
||||
self.seen = True
|
||||
|
||||
parser = Marker()
|
||||
parser.feed(done.stdout)
|
||||
return parser.seen
|
||||
|
||||
|
||||
Browser = Callable[[str, str], bool]
|
||||
|
||||
|
||||
class XSSOracle(OutputOracle):
|
||||
def __init__(self, executor: BrowserExecutor) -> None:
|
||||
def __init__(self, executor: Browser) -> None:
|
||||
self.executor = executor
|
||||
|
||||
def capture(self, output: str) -> None:
|
||||
self.seen |= self.executor(output, self.canary.nonce)
|
||||
|
||||
|
||||
def oracle_for(family: str, browser: BrowserExecutor | None = None) -> Oracle | None:
|
||||
def oracle_for(family: str, browser: Browser | None = None) -> Oracle | None:
|
||||
factories = {"path": PathOracle, "sql": SQLOracle, "command": CommandOracle, "ssrf": SSRFOracle}
|
||||
if family == "xss":
|
||||
return XSSOracle(browser) if browser else None
|
||||
|
||||
Reference in New Issue
Block a user