marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-17 20:19:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

search-with-proof: family-to-oracle mapping, build memory, tolerant tool calls

Browse Source 
Demos declare their oracle within the family allowed set (injection: sql or
command; output_encoding: xss; untrusted_destination: ssrf; path: path);
builds and app starts use the search memory budget instead of the 256 MB tool
limit (npm died with SIGTRAP); malformed tool-call JSON is fed back once.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-05 21:07:58 +02:00
co-authored by Claude Fable 5.1
parent c190d34f60
commit bf243e6680
16 changed files with 334 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openultrasast/search/board.py
+1 -1
View File
@@ -115,7 +115,7 @@ class Board:
state = self.state
view = {key: state[key] for key in ("version", "end_reason", "facts", "intents", "hints")}
view["candidate"] = self.candidate()
view["family"] = view["candidate"].get("oracle", view["candidate"].get("family", "unknown"))
view["family"] = view["candidate"].get("family", "unknown")
view["budget_left"] = budget_left or {}
# Keep recent observations, with explicit truncation, without mutating the board.
remaining = 12000
src/openultrasast/search/coordinator.py
+10 -1
View File
@@ -247,7 +247,15 @@ class Coordinator:
if self.progress["demonstrations"] >= self.budget.demonstrations:
return self._end("budget_spent", "demonstrations")
self.progress["demonstrations"] += 1
result = self._call("verify", {"fact_id": demo["id"], "demo": demo["demo"]})
result = self._call(
"verify",
{
"fact_id": demo["id"],
"demo": demo["demo"],
"family": self.board.candidate().get("family"),
"oracle": demo.get("oracle"),
},
)
self.progress["verified"].append(demo["id"])
refs = result.get("evidence_refs")
if result.get("status") == "ok" and result.get("outcome") == "demonstrated" and self._refs(refs):
@@ -284,6 +292,7 @@ class Coordinator:
)
if isinstance(fact.get("demo"), str) and fact["demo"]:
entry["demo"] = fact["demo"]
entry["oracle"] = fact.get("oracle")
self.state["facts"].append(entry)
else:
intent["failure"] = "invalid_result" if result.get("status") == "ok" else result.get("status", "execution_failure")
src/openultrasast/search/demo.py
+23 -2
View File
@@ -12,6 +12,24 @@ import re
from pathlib import Path, PurePosixPath
from typing import Any, cast
FAMILY_ORACLES = {
"injection": ("sql", "command"),
"path": ("path",),
"output_encoding": ("xss",),
"untrusted_destination": ("ssrf",),
"deserialization": (),
"access_control": (),
"config_secrets": (),
}
def validate_oracle(family: str, oracle: str) -> None:
if not isinstance(family, str) or family not in FAMILY_ORACLES:
raise ValueError("unknown search family")
if oracle not in FAMILY_ORACLES[family]:
raise ValueError("demo oracle is not allowed for search family " + family)
MAX_DEMO_BYTES = 65536
MAX_STEPS = 32
MAX_VALUE_BYTES = 16384
@@ -47,8 +65,9 @@ _CAPTURE_SCHEMA = {
DEMO_SCHEMA = {
"type": "object",
"additionalProperties": False,
"required": ["build", "start", "steps"],
"required": ["oracle", "build", "start", "steps"],
"properties": {
"oracle": {"enum": sorted({o for allowed in FAMILY_ORACLES.values() for o in allowed})},
"build": {
"type": "object",
"additionalProperties": False,
@@ -151,7 +170,9 @@ def validate_demo(value: Any) -> dict[str, Any]:
raise ValueError("demo must be JSON data") from exc
if len(encoded) > MAX_DEMO_BYTES:
raise ValueError("demo exceeds size limit")
_keys(value, {"build", "start", "steps"})
_keys(value, {"oracle", "build", "start", "steps"})
if not isinstance(value["oracle"], str) or not any(value["oracle"] in allowed for allowed in FAMILY_ORACLES.values()):
raise ValueError("unknown demo oracle")
build, start = value["build"], value["start"]
_keys(build, {"recipe", "arguments"})
recipe = build["recipe"]
src/openultrasast/search/executor.py
+7 -5
View File
@@ -31,6 +31,7 @@ from urllib.parse import urlsplit
from ..sandbox import SandboxJob
from . import _sandbox, task_storage
from .budget import SearchBudget
MAX_COMMAND_BYTES = 65536
MAX_RESULT_BYTES = 65536
@@ -198,7 +199,7 @@ class InProcessExecutor:
task_storage.check(task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else self.repo)
self.phase = "run"
if name == "prepare_verification":
return self._prepare_verification(args)
return self._prepare_verification(args, command.limits)
if name == "stop":
self.stopped = True
return {"stopped": True}
@@ -248,7 +249,7 @@ class InProcessExecutor:
self.repo,
{},
max(1, math.ceil(min(timeout, self._remaining()))),
max(1, command.limits.get("memory_bytes", 256 * 1024**2) // 1024**2),
max(1, command.limits.get("memory_bytes", SearchBudget().memory_bytes) // 1024**2),
128,
),
scratch=Path(directory),
@@ -299,7 +300,8 @@ class InProcessExecutor:
# NPROC counts all processes of the real uid, so an absolute 128 blocks builds on busy hosts.
if process_limit is not None:
resource.setrlimit(resource.RLIMIT_NPROC, (process_limit,) * 2)
resource.setrlimit(resource.RLIMIT_AS, (limits.get("memory_bytes", 256 * 1024**2),) * 2)
if "memory_bytes" in limits:
resource.setrlimit(resource.RLIMIT_AS, (limits["memory_bytes"],) * 2)
resource.setrlimit(resource.RLIMIT_CPU, (math.ceil(timeout) + 1,) * 2)
timeout = min(timeout, self._remaining())
@@ -383,7 +385,7 @@ class InProcessExecutor:
"truncated": os.fstat(out.fileno()).st_size > 16384 or os.fstat(err.fileno()).st_size > 16384,
}
def _prepare_verification(self, spec: dict[str, Any]) -> dict[str, Any]:
def _prepare_verification(self, spec: dict[str, Any], limits: dict[str, Any]) -> dict[str, Any]:
"""Build and export through the host-scoped output URL, never a model URL."""
import shutil
@@ -417,7 +419,7 @@ class InProcessExecutor:
# Invoke with sh because checkout archives may lose execute bits.
wrapper = self._path(str(path.relative_to(self.repo) / "gradlew"))
argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"]
built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES})
built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES, **limits})
if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
return {**built, "status": "could_not_build"}
# Validate internal generated links before materializing them as
src/openultrasast/search/probe.py
+8 -1
View File
@@ -21,6 +21,7 @@ from pathlib import Path
from typing import Any
from . import _sandbox
from .demo import FAMILY_ORACLES
from .oracles import BrowserExecutor
from .task_storage import exception_reason
from .verify import Side, verify
@@ -53,6 +54,7 @@ def materialise(root: Path, family: str) -> tuple[tuple[Side, Side], dict[str, P
demo.mkdir()
demos[name] = demo
schema: dict[str, Any] = {
"oracle": family,
"build": {"recipe": "none", "arguments": []},
"start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
"steps": [{"type": "cli", "arguments": [family, PAYLOADS[family]]}],
@@ -121,7 +123,12 @@ def probe(root: Path) -> dict[str, Any]:
results: dict[str, Any] = {}
record["oracles"][family] = results
for name, demo in demos.items():
result = verify(*sides, demo, family, browser=BrowserExecutor() if family == "xss" else None)
result = verify(
*sides,
demo,
next(f for f, kinds in FAMILY_ORACLES.items() if family in kinds),
browser=BrowserExecutor() if family == "xss" else None,
)
results[name] = asdict(result)
# SSRF/browser need additional namespace capabilities beyond the
# generic preflight. These failures must not look like oracle misses.
src/openultrasast/search/verify.py
+21 -11
View File
@@ -27,7 +27,8 @@ from typing import Any, cast
from ..sandbox import SandboxJob, SandboxResult
from . import _sandbox, task_storage
from .demo import BUILD_RECIPES, RUNTIMES, capture_output, load_demo, substitute
from .budget import SearchBudget
from .demo import BUILD_RECIPES, FAMILY_ORACLES, RUNTIMES, capture_output, load_demo, substitute, validate_oracle
from .oracles import BrowserExecutor, Canary, Oracle, oracle_for
from .task_storage import exception_reason
@@ -277,7 +278,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
mounts = {f"/deps/{i}": p for i, p in enumerate(dependencies)}
def job(command: tuple[str, ...], checkout: Path = checkout) -> SandboxJob:
return SandboxJob("", command, checkout, {}, timeout, 256, 128)
return SandboxJob("", command, checkout, {}, timeout, SearchBudget().memory_bytes // 1024**2, 128)
products = root / "products"
if side.products is not None:
@@ -448,7 +449,7 @@ def _run_task(
def limits() -> None:
resource.setrlimit(resource.RLIMIT_FSIZE, (1024 * 1024,) * 2)
resource.setrlimit(resource.RLIMIT_AS, (job.memory_mb * 1024**2,) * 2)
# The external task memory limit bounds app starts; RLIMIT_AS breaks V8 reservations.
resource.setrlimit(resource.RLIMIT_CPU, (job.timeout_seconds + 1,) * 2)
# Driver JSON includes command paths, but arbitrary user input stays data.
@@ -481,13 +482,15 @@ def verify_side_task(side: Side, demo: Path, family: str, timeout_seconds: int =
This explicit API is never selected as a fallback in the host verifier.
"""
load_demo(demo)
schema = load_demo(demo)
if family in FAMILY_ORACLES and not FAMILY_ORACLES[family]:
return SideRecord("no_oracle", (), 0, "no owned oracle", isolation_mode="task-boundary")
validate_oracle(family, schema["oracle"])
kind = schema["oracle"]
if timeout_seconds < 1:
raise ValueError("timeout must be positive")
browser = BrowserExecutor(task_boundary=True) if family == "xss" else None
if oracle_for(family, browser) is None:
return SideRecord("no_oracle", (), 0, "no owned oracle", isolation_mode="task-boundary")
return _side(side, demo, family, timeout_seconds, browser, task_boundary=True)
browser = BrowserExecutor(task_boundary=True) if kind == "xss" else None
return _side(side, demo, kind, timeout_seconds, browser, task_boundary=True)
def verify(
@@ -497,17 +500,24 @@ def verify(
family: str,
*,
timeout_seconds: int = 5,
oracle: str | None = None,
browser: BrowserExecutor | None = None,
task_dispatcher: Callable[..., SideRecord] | None = None,
) -> VerificationRecord:
"""Affected is vulnerable/head; safe is fixed/base. No negative verdict exists."""
started = time.monotonic()
if oracle_for(family, browser) is None:
if family in FAMILY_ORACLES and not FAMILY_ORACLES[family]:
return VerificationRecord("no_oracle", (), time.monotonic() - started, "no configured owned oracle for " + family)
if timeout_seconds < 1:
raise ValueError("timeout must be positive")
try:
load_demo(artefact)
schema = load_demo(artefact)
validate_oracle(family, schema["oracle"])
if oracle is not None and oracle != schema["oracle"]:
raise ValueError("oracle differs from demo")
kind = schema["oracle"]
if oracle_for(kind, browser) is None:
raise ValueError("oracle runtime unavailable")
except (OSError, ValueError) as exc:
return VerificationRecord("inconclusive", (), time.monotonic() - started, "invalid declarative demo: " + exception_reason(exc))
mode = "task-boundary" if task_dispatcher is not None else _sandbox.isolation_mode()
@@ -527,7 +537,7 @@ def verify(
if any(s.isolation_mode != "task-boundary" for s in sides):
raise _sandbox.IsolationUnavailable("dispatcher did not attest task-boundary isolation")
else:
sides = tuple(_side(s, artefact, family, timeout_seconds, browser) for s in (affected, safe))
sides = tuple(_side(s, artefact, kind, timeout_seconds, browser) for s in (affected, safe))
if any(s.outcome == "observed" and len(s.runs) != 3 for s in sides):
outcome, reason = "inconclusive", "each side requires exactly three fresh observations"
elif sides[1].outcome != "observed":
src/openultrasast/search/verify_task.py
+2 -3
View File
@@ -20,7 +20,7 @@ from typing import Any, cast
from urllib.parse import urlsplit
from . import task_storage
from .demo import validate_demo
from .demo import validate_demo, validate_oracle
from .executor import clean_environment
from .verify import Side, SideRecord
@@ -132,8 +132,7 @@ def validate_spec(spec: Any) -> dict[str, Any]:
validate_demo(spec["demo"])
if spec["demo"]["build"]["recipe"] != "none":
raise ValueError("verifier requires executor-built archive; builds are forbidden")
if spec["family"] not in {"sql", "path", "command", "ssrf", "xss"}:
raise ValueError("invalid verification family")
validate_oracle(spec["family"], spec["demo"]["oracle"])
timeout = spec["timeout_seconds"]
if type(timeout) is not int or not math.isfinite(timeout) or not 1 <= timeout <= 300:
raise ValueError("invalid verification timeout")
src/openultrasast/search/worker.py
+31 -10
View File
@@ -17,8 +17,9 @@ from ..model.endpoint import Prices
from ..plane.budget import BudgetExhausted, SpendBudget, cost_of
from ..provider.openrouter import OpenRouterChatClient, parse_json_content
from . import _sandbox
from .budget import SearchBudget
from .coordinator import SearchTask
from .demo import DEMO_SCHEMA, load_demo, validate_demo
from .demo import DEMO_SCHEMA, FAMILY_ORACLES, load_demo, validate_demo, validate_oracle
from .executor import ObjectStoreExecutor
from .task_storage import exception_reason
@@ -77,7 +78,7 @@ TOOLS = [
DEMO_GUIDE = (
'Demo schema: {"build":{"recipe":"none","arguments":[]},'
'Demo schema: {"oracle":"path","build":{"recipe":"none","arguments":[]},'
'"start":{"runtime":"python","path":"app.py","arguments":[],"mode":"cli"},'
'"steps":[{"type":"cli","arguments":["hello"]}]}. This is a minimal example, not evidence. '
"Recipes: none,pip,npm,composer,maven,gradle; none takes [], others one checkout-relative manifest/project path. "
@@ -94,8 +95,10 @@ ORACLE_GUIDE = (
"SSRF provides a private loopback listener with a fresh nonce URL and observes a request to it. "
"XSS installs a verifier-owned alert hook in a browser and observes actual execution of alert('ousast-xss'); "
"reflection alone is insufficient. "
"output_encoding maps to XSS; untrusted_destination maps to SSRF. Injection needs an explicit SQL or command subtype. "
"Other families have no owned oracle; report that gap instead of inventing one."
"Set the required oracle field using the allowed choices below; for injection choose sql or command. "
"Other families have no owned oracle; report that gap instead of inventing one. "
+ "Allowed family/oracle choices: "
+ json.dumps(FAMILY_ORACLES)
)
REASON_PROMPT = (
"Your goal is a declarative demo conforming to demo.py's schema that makes the verifier observe the effect "
@@ -188,7 +191,7 @@ class Worker:
raise ValueError("demo symlinks refused")
self.max_output_tokens, self.max_steps = max_output_tokens, max_steps
self.deadline = float("inf")
self.limits: dict[str, Any] = {"memory_bytes": 256 * 1024**2, "disk_bytes": 128 * 1024**2}
self.limits: dict[str, Any] = {"memory_bytes": SearchBudget().memory_bytes, "disk_bytes": 128 * 1024**2}
self.evidence: set[str] = set()
def _remaining(self) -> int:
@@ -270,7 +273,10 @@ class Worker:
name,
args,
timeout_seconds=timeout,
limits={key: value for key, value in self.limits.items() if key in {"memory_bytes", "disk_bytes", "result_bytes"}},
limits={
**{key: value for key, value in self.limits.items() if key in {"memory_bytes", "disk_bytes", "result_bytes"}},
**({"memory_bytes": min(256 * 1024**2, self.limits.get("memory_bytes", 256 * 1024**2))} if name != "run" else {}),
},
)
if name == "write_demo":
if set(args) != {"schema"}:
@@ -339,6 +345,7 @@ class Worker:
]
self.evidence = set()
self.observations = []
malformed = 0
for index in range(self.max_steps):
messages[0]["content"] = (
EXPLORE_PROMPT
@@ -354,9 +361,19 @@ class Worker:
raise ValueError("explore must use tools or finish")
for call in calls:
function = call["function"]
name, args = function["name"], json.loads(function["arguments"])
if not isinstance(args, dict):
raise ValueError("tool arguments must be an object")
name = function["name"]
try:
args = json.loads(function["arguments"])
if not isinstance(args, dict):
raise ValueError("tool arguments must be an object")
except (ValueError, TypeError) as exc:
malformed += 1
error = "invalid JSON arguments: " + str(exc)[:500]
messages.append({"role": "tool", "tool_call_id": call["id"], "content": json.dumps({"error": error})})
if malformed >= 2:
return {"status": "execution_failure", "failure": "two consecutive malformed tool calls: " + error}
continue
malformed = 0
if name == "finish":
if len(calls) != 1 or len(args) != 1:
raise ValueError("finish must be the only call and return fact OR failure")
@@ -375,7 +392,11 @@ class Worker:
if "demo" in fact:
if fact["demo"] != "demo/":
raise ValueError("incomplete demonstration")
load_demo(self.demo)
schema = load_demo(self.demo)
family = (yaml.safe_load(task.snapshot) or {}).get("family")
if family not in (None, "unknown"):
validate_oracle(family, schema["oracle"])
fact["oracle"] = schema["oracle"]
return {"fact": fact}
try:
result = self.tool(name, args)