search-with-proof: family-to-oracle mapping, build memory, tolerant tool calls

Demos declare their oracle within the family allowed set (injection: sql or
command; output_encoding: xss; untrusted_destination: ssrf; path: path);
builds and app starts use the search memory budget instead of the 256 MB tool
limit (npm died with SIGTRAP); malformed tool-call JSON is fed back once.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/search/pilot_run.py

@@ -36,9 +36,9 @@ from openultrasast.search import _sandbox
 from openultrasast.search.board import Board
 from openultrasast.search.budget import SearchBudget
 from openultrasast.search.coordinator import Coordinator
-from openultrasast.search.demo import load_demo
+from openultrasast.search.demo import FAMILY_ORACLES, load_demo, validate_oracle
 from openultrasast.search.executor import InProcessExecutor
-from openultrasast.search.oracles import BrowserExecutor, oracle_for
+from openultrasast.search.oracles import BrowserExecutor
 from openultrasast.search.probe import materialise
 from openultrasast.search.task_storage import diagnostic, exception_reason
 from openultrasast.search.verify import Side, VerificationRecord, verify
@@ -119,10 +119,7 @@ class Pilot:
             run_budget=run_budget,
             max_output_tokens=args.max_tokens,
         )
-        self.family = {"output_encoding": "xss", "untrusted_destination": "ssrf"}.get(row["family"], row["family"])
-        # Injection is ambiguous without a manifest oracle subtype; never guess SQL vs command.
-        if self.family == "injection":
-            self.family = row.get("oracle", "injection")
+        self.family = row["family"]
         self.sides = None
         self.verifications = []
 
@@ -220,13 +217,13 @@ class Pilot:
             wall["build_export"] = wall.get("build_export", 0) + time.monotonic() - before
 
     def verification(self):
-        demo = load_demo(self.root / "demo")
-        browser = BrowserExecutor(task_boundary=self.args.verify_lane == "ax") if self.family == "xss" else None
-        if oracle_for(self.family, browser) is None:
+        if not FAMILY_ORACLES[self.family]:
             return VerificationRecord("no_oracle", (), 0, "no configured owned oracle")
+        demo = load_demo(self.root / "demo")
+        validate_oracle(self.family, demo["oracle"])
         if self.args.dry_run:
             sides = self.sides if self.args.side == "vulnerable" else (self.sides[1], self.sides[1])
-            return verify(*sides, self.root / "demo", self.family)
+            return verify(*sides, self.root / "demo", self.family, oracle=demo["oracle"])
         revisions = [self.row[self.args.side], self.row["fixed"]]
         if self.args.verify_lane == "ax":
             # Side paths are opaque lookup keys; no checkout or opposite revision reaches the brain.
@@ -251,8 +248,9 @@ class Pilot:
                 *sides,
                 self.root / "demo",
                 self.family,
+                oracle=demo["oracle"],
                 task_dispatcher=dispatcher,
-                browser=BrowserExecutor(task_boundary=True) if self.family == "xss" else None,
+                browser=BrowserExecutor(task_boundary=True) if demo["oracle"] == "xss" else None,
             )
         if _sandbox.isolation_mode() != "userns":
             raise _sandbox.IsolationUnavailable("VM verification requires userns")
@@ -270,7 +268,13 @@ class Pilot:
             if built_spec != {**spec, "demo": expected_demo}:
                 raise ValueError("executor altered verification contract")
             (self.root / "verify-demo/demo.json").write_text(json.dumps(built_spec["demo"]))
-        return verify(*sides, self.root / "verify-demo", self.family, browser=BrowserExecutor() if self.family == "xss" else None)
+        return verify(
+            *sides,
+            self.root / "verify-demo",
+            self.family,
+            oracle=demo["oracle"],
+            browser=BrowserExecutor() if demo["oracle"] == "xss" else None,
+        )
 
     def dispatch(self, task):
         self.phase = task.step
@@ -341,7 +345,7 @@ class Pilot:
 
     def run(self):
         started = time.monotonic()
-        coordinator = Coordinator(self.board, self.dispatch, budget=SearchBudget(retries=0, memory_bytes=2 * 1024**3))
+        coordinator = Coordinator(self.board, self.dispatch, budget=SearchBudget(retries=0))
         try:
             state = coordinator.run()
             self.record["end_reason"] = state["end_reason"]

src/openultrasast/search/board.py

@@ -115,7 +115,7 @@ class Board:
         state = self.state
         view = {key: state[key] for key in ("version", "end_reason", "facts", "intents", "hints")}
         view["candidate"] = self.candidate()
-        view["family"] = view["candidate"].get("oracle", view["candidate"].get("family", "unknown"))
+        view["family"] = view["candidate"].get("family", "unknown")
         view["budget_left"] = budget_left or {}
         # Keep recent observations, with explicit truncation, without mutating the board.
         remaining = 12000

src/openultrasast/search/coordinator.py

@@ -247,7 +247,15 @@ class Coordinator:
                     if self.progress["demonstrations"] >= self.budget.demonstrations:
                         return self._end("budget_spent", "demonstrations")
                     self.progress["demonstrations"] += 1
-                    result = self._call("verify", {"fact_id": demo["id"], "demo": demo["demo"]})
+                    result = self._call(
+                        "verify",
+                        {
+                            "fact_id": demo["id"],
+                            "demo": demo["demo"],
+                            "family": self.board.candidate().get("family"),
+                            "oracle": demo.get("oracle"),
+                        },
+                    )
                     self.progress["verified"].append(demo["id"])
                     refs = result.get("evidence_refs")
                     if result.get("status") == "ok" and result.get("outcome") == "demonstrated" and self._refs(refs):
@@ -284,6 +292,7 @@ class Coordinator:
                         )
                         if isinstance(fact.get("demo"), str) and fact["demo"]:
                             entry["demo"] = fact["demo"]
+                            entry["oracle"] = fact.get("oracle")
                         self.state["facts"].append(entry)
                     else:
                         intent["failure"] = "invalid_result" if result.get("status") == "ok" else result.get("status", "execution_failure")

src/openultrasast/search/demo.py

@@ -12,6 +12,24 @@ import re
 from pathlib import Path, PurePosixPath
 from typing import Any, cast
 
+FAMILY_ORACLES = {
+    "injection": ("sql", "command"),
+    "path": ("path",),
+    "output_encoding": ("xss",),
+    "untrusted_destination": ("ssrf",),
+    "deserialization": (),
+    "access_control": (),
+    "config_secrets": (),
+}
+
+
+def validate_oracle(family: str, oracle: str) -> None:
+    if not isinstance(family, str) or family not in FAMILY_ORACLES:
+        raise ValueError("unknown search family")
+    if oracle not in FAMILY_ORACLES[family]:
+        raise ValueError("demo oracle is not allowed for search family " + family)
+
+
 MAX_DEMO_BYTES = 65536
 MAX_STEPS = 32
 MAX_VALUE_BYTES = 16384
@@ -47,8 +65,9 @@ _CAPTURE_SCHEMA = {
 DEMO_SCHEMA = {
     "type": "object",
     "additionalProperties": False,
-    "required": ["build", "start", "steps"],
+    "required": ["oracle", "build", "start", "steps"],
     "properties": {
+        "oracle": {"enum": sorted({o for allowed in FAMILY_ORACLES.values() for o in allowed})},
         "build": {
             "type": "object",
             "additionalProperties": False,
@@ -151,7 +170,9 @@ def validate_demo(value: Any) -> dict[str, Any]:
         raise ValueError("demo must be JSON data") from exc
     if len(encoded) > MAX_DEMO_BYTES:
         raise ValueError("demo exceeds size limit")
-    _keys(value, {"build", "start", "steps"})
+    _keys(value, {"oracle", "build", "start", "steps"})
+    if not isinstance(value["oracle"], str) or not any(value["oracle"] in allowed for allowed in FAMILY_ORACLES.values()):
+        raise ValueError("unknown demo oracle")
     build, start = value["build"], value["start"]
     _keys(build, {"recipe", "arguments"})
     recipe = build["recipe"]

src/openultrasast/search/executor.py

@@ -31,6 +31,7 @@ from urllib.parse import urlsplit
 
 from ..sandbox import SandboxJob
 from . import _sandbox, task_storage
+from .budget import SearchBudget
 
 MAX_COMMAND_BYTES = 65536
 MAX_RESULT_BYTES = 65536
@@ -198,7 +199,7 @@ class InProcessExecutor:
             task_storage.check(task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else self.repo)
             self.phase = "run"
         if name == "prepare_verification":
-            return self._prepare_verification(args)
+            return self._prepare_verification(args, command.limits)
         if name == "stop":
             self.stopped = True
             return {"stopped": True}
@@ -248,7 +249,7 @@ class InProcessExecutor:
                     self.repo,
                     {},
                     max(1, math.ceil(min(timeout, self._remaining()))),
-                    max(1, command.limits.get("memory_bytes", 256 * 1024**2) // 1024**2),
+                    max(1, command.limits.get("memory_bytes", SearchBudget().memory_bytes) // 1024**2),
                     128,
                 ),
                 scratch=Path(directory),
@@ -299,7 +300,8 @@ class InProcessExecutor:
             # NPROC counts all processes of the real uid, so an absolute 128 blocks builds on busy hosts.
             if process_limit is not None:
                 resource.setrlimit(resource.RLIMIT_NPROC, (process_limit,) * 2)
-            resource.setrlimit(resource.RLIMIT_AS, (limits.get("memory_bytes", 256 * 1024**2),) * 2)
+            if "memory_bytes" in limits:
+                resource.setrlimit(resource.RLIMIT_AS, (limits["memory_bytes"],) * 2)
             resource.setrlimit(resource.RLIMIT_CPU, (math.ceil(timeout) + 1,) * 2)
 
         timeout = min(timeout, self._remaining())
@@ -383,7 +385,7 @@ class InProcessExecutor:
                 "truncated": os.fstat(out.fileno()).st_size > 16384 or os.fstat(err.fileno()).st_size > 16384,
             }
 
-    def _prepare_verification(self, spec: dict[str, Any]) -> dict[str, Any]:
+    def _prepare_verification(self, spec: dict[str, Any], limits: dict[str, Any]) -> dict[str, Any]:
         """Build and export through the host-scoped output URL, never a model URL."""
         import shutil
 
@@ -417,7 +419,7 @@ class InProcessExecutor:
                     # Invoke with sh because checkout archives may lose execute bits.
                     wrapper = self._path(str(path.relative_to(self.repo) / "gradlew"))
                     argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"]
-                built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES})
+                built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES, **limits})
                 if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
                     return {**built, "status": "could_not_build"}
             # Validate internal generated links before materializing them as

src/openultrasast/search/probe.py

@@ -21,6 +21,7 @@ from pathlib import Path
 from typing import Any
 
 from . import _sandbox
+from .demo import FAMILY_ORACLES
 from .oracles import BrowserExecutor
 from .task_storage import exception_reason
 from .verify import Side, verify
@@ -53,6 +54,7 @@ def materialise(root: Path, family: str) -> tuple[tuple[Side, Side], dict[str, P
         demo.mkdir()
         demos[name] = demo
     schema: dict[str, Any] = {
+        "oracle": family,
         "build": {"recipe": "none", "arguments": []},
         "start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
         "steps": [{"type": "cli", "arguments": [family, PAYLOADS[family]]}],
@@ -121,7 +123,12 @@ def probe(root: Path) -> dict[str, Any]:
             results: dict[str, Any] = {}
             record["oracles"][family] = results
             for name, demo in demos.items():
-                result = verify(*sides, demo, family, browser=BrowserExecutor() if family == "xss" else None)
+                result = verify(
+                    *sides,
+                    demo,
+                    next(f for f, kinds in FAMILY_ORACLES.items() if family in kinds),
+                    browser=BrowserExecutor() if family == "xss" else None,
+                )
                 results[name] = asdict(result)
                 # SSRF/browser need additional namespace capabilities beyond the
                 # generic preflight. These failures must not look like oracle misses.

src/openultrasast/search/verify.py

@@ -27,7 +27,8 @@ from typing import Any, cast
 
 from ..sandbox import SandboxJob, SandboxResult
 from . import _sandbox, task_storage
-from .demo import BUILD_RECIPES, RUNTIMES, capture_output, load_demo, substitute
+from .budget import SearchBudget
+from .demo import BUILD_RECIPES, FAMILY_ORACLES, RUNTIMES, capture_output, load_demo, substitute, validate_oracle
 from .oracles import BrowserExecutor, Canary, Oracle, oracle_for
 from .task_storage import exception_reason
 
@@ -277,7 +278,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe
                 mounts = {f"/deps/{i}": p for i, p in enumerate(dependencies)}
 
                 def job(command: tuple[str, ...], checkout: Path = checkout) -> SandboxJob:
-                    return SandboxJob("", command, checkout, {}, timeout, 256, 128)
+                    return SandboxJob("", command, checkout, {}, timeout, SearchBudget().memory_bytes // 1024**2, 128)
 
                 products = root / "products"
                 if side.products is not None:
@@ -448,7 +449,7 @@ def _run_task(
 
     def limits() -> None:
         resource.setrlimit(resource.RLIMIT_FSIZE, (1024 * 1024,) * 2)
-        resource.setrlimit(resource.RLIMIT_AS, (job.memory_mb * 1024**2,) * 2)
+        # The external task memory limit bounds app starts; RLIMIT_AS breaks V8 reservations.
         resource.setrlimit(resource.RLIMIT_CPU, (job.timeout_seconds + 1,) * 2)
 
     # Driver JSON includes command paths, but arbitrary user input stays data.
@@ -481,13 +482,15 @@ def verify_side_task(side: Side, demo: Path, family: str, timeout_seconds: int =
 
     This explicit API is never selected as a fallback in the host verifier.
     """
-    load_demo(demo)
+    schema = load_demo(demo)
+    if family in FAMILY_ORACLES and not FAMILY_ORACLES[family]:
+        return SideRecord("no_oracle", (), 0, "no owned oracle", isolation_mode="task-boundary")
+    validate_oracle(family, schema["oracle"])
+    kind = schema["oracle"]
     if timeout_seconds < 1:
         raise ValueError("timeout must be positive")
-    browser = BrowserExecutor(task_boundary=True) if family == "xss" else None
-    if oracle_for(family, browser) is None:
-        return SideRecord("no_oracle", (), 0, "no owned oracle", isolation_mode="task-boundary")
-    return _side(side, demo, family, timeout_seconds, browser, task_boundary=True)
+    browser = BrowserExecutor(task_boundary=True) if kind == "xss" else None
+    return _side(side, demo, kind, timeout_seconds, browser, task_boundary=True)
 
 
 def verify(
@@ -497,17 +500,24 @@ def verify(
     family: str,
     *,
     timeout_seconds: int = 5,
+    oracle: str | None = None,
     browser: BrowserExecutor | None = None,
     task_dispatcher: Callable[..., SideRecord] | None = None,
 ) -> VerificationRecord:
     """Affected is vulnerable/head; safe is fixed/base. No negative verdict exists."""
     started = time.monotonic()
-    if oracle_for(family, browser) is None:
+    if family in FAMILY_ORACLES and not FAMILY_ORACLES[family]:
         return VerificationRecord("no_oracle", (), time.monotonic() - started, "no configured owned oracle for " + family)
     if timeout_seconds < 1:
         raise ValueError("timeout must be positive")
     try:
-        load_demo(artefact)
+        schema = load_demo(artefact)
+        validate_oracle(family, schema["oracle"])
+        if oracle is not None and oracle != schema["oracle"]:
+            raise ValueError("oracle differs from demo")
+        kind = schema["oracle"]
+        if oracle_for(kind, browser) is None:
+            raise ValueError("oracle runtime unavailable")
     except (OSError, ValueError) as exc:
         return VerificationRecord("inconclusive", (), time.monotonic() - started, "invalid declarative demo: " + exception_reason(exc))
     mode = "task-boundary" if task_dispatcher is not None else _sandbox.isolation_mode()
@@ -527,7 +537,7 @@ def verify(
         if any(s.isolation_mode != "task-boundary" for s in sides):
             raise _sandbox.IsolationUnavailable("dispatcher did not attest task-boundary isolation")
     else:
-        sides = tuple(_side(s, artefact, family, timeout_seconds, browser) for s in (affected, safe))
+        sides = tuple(_side(s, artefact, kind, timeout_seconds, browser) for s in (affected, safe))
     if any(s.outcome == "observed" and len(s.runs) != 3 for s in sides):
         outcome, reason = "inconclusive", "each side requires exactly three fresh observations"
     elif sides[1].outcome != "observed":

src/openultrasast/search/verify_task.py

@@ -20,7 +20,7 @@ from typing import Any, cast
 from urllib.parse import urlsplit
 
 from . import task_storage
-from .demo import validate_demo
+from .demo import validate_demo, validate_oracle
 from .executor import clean_environment
 from .verify import Side, SideRecord
 
@@ -132,8 +132,7 @@ def validate_spec(spec: Any) -> dict[str, Any]:
     validate_demo(spec["demo"])
     if spec["demo"]["build"]["recipe"] != "none":
         raise ValueError("verifier requires executor-built archive; builds are forbidden")
-    if spec["family"] not in {"sql", "path", "command", "ssrf", "xss"}:
-        raise ValueError("invalid verification family")
+    validate_oracle(spec["family"], spec["demo"]["oracle"])
     timeout = spec["timeout_seconds"]
     if type(timeout) is not int or not math.isfinite(timeout) or not 1 <= timeout <= 300:
         raise ValueError("invalid verification timeout")

src/openultrasast/search/worker.py

@@ -17,8 +17,9 @@ from ..model.endpoint import Prices
 from ..plane.budget import BudgetExhausted, SpendBudget, cost_of
 from ..provider.openrouter import OpenRouterChatClient, parse_json_content
 from . import _sandbox
+from .budget import SearchBudget
 from .coordinator import SearchTask
-from .demo import DEMO_SCHEMA, load_demo, validate_demo
+from .demo import DEMO_SCHEMA, FAMILY_ORACLES, load_demo, validate_demo, validate_oracle
 from .executor import ObjectStoreExecutor
 from .task_storage import exception_reason
 
@@ -77,7 +78,7 @@ TOOLS = [
 
 
 DEMO_GUIDE = (
-    'Demo schema: {"build":{"recipe":"none","arguments":[]},'
+    'Demo schema: {"oracle":"path","build":{"recipe":"none","arguments":[]},'
     '"start":{"runtime":"python","path":"app.py","arguments":[],"mode":"cli"},'
     '"steps":[{"type":"cli","arguments":["hello"]}]}. This is a minimal example, not evidence. '
     "Recipes: none,pip,npm,composer,maven,gradle; none takes [], others one checkout-relative manifest/project path. "
@@ -94,8 +95,10 @@ ORACLE_GUIDE = (
     "SSRF provides a private loopback listener with a fresh nonce URL and observes a request to it. "
     "XSS installs a verifier-owned alert hook in a browser and observes actual execution of alert('ousast-xss'); "
     "reflection alone is insufficient. "
-    "output_encoding maps to XSS; untrusted_destination maps to SSRF. Injection needs an explicit SQL or command subtype. "
-    "Other families have no owned oracle; report that gap instead of inventing one."
+    "Set the required oracle field using the allowed choices below; for injection choose sql or command. "
+    "Other families have no owned oracle; report that gap instead of inventing one. "
+    + "Allowed family/oracle choices: "
+    + json.dumps(FAMILY_ORACLES)
 )
 REASON_PROMPT = (
     "Your goal is a declarative demo conforming to demo.py's schema that makes the verifier observe the effect "
@@ -188,7 +191,7 @@ class Worker:
             raise ValueError("demo symlinks refused")
         self.max_output_tokens, self.max_steps = max_output_tokens, max_steps
         self.deadline = float("inf")
-        self.limits: dict[str, Any] = {"memory_bytes": 256 * 1024**2, "disk_bytes": 128 * 1024**2}
+        self.limits: dict[str, Any] = {"memory_bytes": SearchBudget().memory_bytes, "disk_bytes": 128 * 1024**2}
         self.evidence: set[str] = set()
 
     def _remaining(self) -> int:
@@ -270,7 +273,10 @@ class Worker:
                 name,
                 args,
                 timeout_seconds=timeout,
-                limits={key: value for key, value in self.limits.items() if key in {"memory_bytes", "disk_bytes", "result_bytes"}},
+                limits={
+                    **{key: value for key, value in self.limits.items() if key in {"memory_bytes", "disk_bytes", "result_bytes"}},
+                    **({"memory_bytes": min(256 * 1024**2, self.limits.get("memory_bytes", 256 * 1024**2))} if name != "run" else {}),
+                },
             )
         if name == "write_demo":
             if set(args) != {"schema"}:
@@ -339,6 +345,7 @@ class Worker:
         ]
         self.evidence = set()
         self.observations = []
+        malformed = 0
         for index in range(self.max_steps):
             messages[0]["content"] = (
                 EXPLORE_PROMPT
@@ -354,9 +361,19 @@ class Worker:
                 raise ValueError("explore must use tools or finish")
             for call in calls:
                 function = call["function"]
-                name, args = function["name"], json.loads(function["arguments"])
-                if not isinstance(args, dict):
-                    raise ValueError("tool arguments must be an object")
+                name = function["name"]
+                try:
+                    args = json.loads(function["arguments"])
+                    if not isinstance(args, dict):
+                        raise ValueError("tool arguments must be an object")
+                except (ValueError, TypeError) as exc:
+                    malformed += 1
+                    error = "invalid JSON arguments: " + str(exc)[:500]
+                    messages.append({"role": "tool", "tool_call_id": call["id"], "content": json.dumps({"error": error})})
+                    if malformed >= 2:
+                        return {"status": "execution_failure", "failure": "two consecutive malformed tool calls: " + error}
+                    continue
+                malformed = 0
                 if name == "finish":
                     if len(calls) != 1 or len(args) != 1:
                         raise ValueError("finish must be the only call and return fact OR failure")
@@ -375,7 +392,11 @@ class Worker:
                     if "demo" in fact:
                         if fact["demo"] != "demo/":
                             raise ValueError("incomplete demonstration")
-                        load_demo(self.demo)
+                        schema = load_demo(self.demo)
+                        family = (yaml.safe_load(task.snapshot) or {}).get("family")
+                        if family not in (None, "unknown"):
+                            validate_oracle(family, schema["oracle"])
+                        fact["oracle"] = schema["oracle"]
                     return {"fact": fact}
                 try:
                     result = self.tool(name, args)

tests/test_search_board.py

@@ -65,7 +65,7 @@ def test_snapshot_caps_fact_text_and_preserves_context(tmp_path):
         ],
     )
     view = yaml.safe_load(board.snapshot(budget_left={"tasks": 3}))
-    assert view["candidate"]["file"] == "app.py" and view["family"] == "sql"
+    assert view["candidate"]["file"] == "app.py" and view["family"] == "injection"
     assert view["budget_left"]["tasks"] == 3
     assert {i["status"] for i in view["intents"]} == {"open", "concluded"}
     assert all(len(f["text"]) <= 2000 for f in view["facts"])

tests/test_search_demo.py

@@ -7,6 +7,7 @@ from openultrasast.search.demo import load_demo, validate_demo
 
 def document():
     return {
+        "oracle": "path",
         "build": {"recipe": "none", "arguments": []},
         "start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
         "steps": [{"type": "cli", "arguments": ["hello"]}],
@@ -75,3 +76,38 @@ def test_manifest_recipe_requires_file_path(recipe):
     value["build"] = {"recipe": recipe, "arguments": ["."]}
     with pytest.raises(ValueError):
         validate_demo(value)
+
+
+@pytest.mark.parametrize("oracle", [None, "injection", "unknown", [], 1])
+def test_oracle_required_and_known(oracle):
+    value = document()
+    if oracle is None:
+        value.pop("oracle")
+    else:
+        value["oracle"] = oracle
+    with pytest.raises(ValueError):
+        validate_demo(value)
+
+
+@pytest.mark.parametrize(
+    "family,allowed",
+    [
+        ("injection", ("sql", "command")),
+        ("path", ("path",)),
+        ("output_encoding", ("xss",)),
+        ("untrusted_destination", ("ssrf",)),
+        ("deserialization", ()),
+        ("access_control", ()),
+        ("config_secrets", ()),
+    ],
+)
+def test_family_oracle_contract(family, allowed):
+    from openultrasast.search.demo import FAMILY_ORACLES, validate_oracle
+
+    assert FAMILY_ORACLES[family] == allowed
+    for oracle in ("sql", "command", "path", "xss", "ssrf"):
+        if oracle in allowed:
+            validate_oracle(family, oracle)
+        else:
+            with pytest.raises(ValueError, match="not allowed"):
+                validate_oracle(family, oracle)

tests/test_search_pilot_run.py

@@ -247,7 +247,7 @@ def test_no_oracle_does_not_acquire_or_build(tmp_path, monkeypatch):
     pilot = make_pilot(tmp_path)
     _, demos = materialise(tmp_path / "probe", "path")
     (tmp_path / "demo/demo.json").write_bytes((demos["real"] / "demo.json").read_bytes())
-    pilot.family = "injection"
+    pilot.family = "deserialization"
     monkeypatch.setattr(pilot, "prepare", lambda *args: pytest.fail("no-oracle must not build"))
     assert pilot.verification().outcome == "no_oracle"
     assert pilot.record["tasks_submitted"] == 0
@@ -467,3 +467,16 @@ def test_live_driver_stops_remaining_pairs_on_infrastructure_failure(tmp_path, m
     assert summary["end_reason"] == "instrument_failure"
     assert summary["aborted_reason"] == search["aborted_reason"]
     assert "running=0" in summary["aborted_reason"]
+
+
+@pytest.mark.parametrize("oracle", ["sql", "command"])
+def test_injection_pilot_uses_demo_choice_without_manifest_subtype(tmp_path, oracle):
+    from openultrasast.search.probe import materialise
+
+    pilot = make_pilot(tmp_path, side="vulnerable", dry_run=True)
+    pilot.family = "injection"
+    pilot.row.pop("oracle", None)
+    pilot.sides, demos = materialise(tmp_path / "probe", oracle)
+    (tmp_path / "demo/demo.json").write_bytes((demos["real"] / "demo.json").read_bytes())
+    result = pilot.verification()
+    assert result.outcome == "demonstrated", result

tests/test_search_task_storage.py

@@ -16,6 +16,7 @@ from openultrasast.search.verify import Side, verify_side_task
 def spec():
     return {
         "demo": {
+            "oracle": "path",
             "build": {"recipe": "none", "arguments": []},
             "start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
             "steps": [{"type": "cli", "arguments": []}],
@@ -281,3 +282,38 @@ def test_preparation_with_many_host_processes(workspace, monkeypatch):
             os.kill(pid, signal.SIGKILL)
         for pid in children:
             os.waitpid(pid, 0)
+
+
+@pytest.mark.parametrize("memory_bytes", [None, 256 * 1024**2, 5 * 1024**3])
+def test_prepare_build_virtual_memory(workspace, monkeypatch, memory_bytes):
+    from openultrasast.search.demo import BUILD_RECIPES
+
+    repo = workspace / "checkout"
+    repo.mkdir()
+    source = "import mmap; region=mmap.mmap(-1,512*1024**2); print(len(region))"
+    (repo / "app.py").write_text(source)
+    print("input_bytes=", len((repo / "app.py").read_bytes()))
+    monkeypatch.setitem(BUILD_RECIPES, "npm", (sys.executable, "-c", source))
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    blobs = []
+    monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
+    request = spec()
+    request["demo"]["build"] = {"recipe": "npm", "arguments": ["."]}
+    limits = {} if memory_bytes is None else {"memory_bytes": memory_bytes}
+    result = app.submit("prepare_verification", request, limits=limits)
+    if memory_bytes == 256 * 1024**2:
+        assert result["status"] == "could_not_build" and not blobs
+    else:
+        assert result["status"] == "ok" and blobs, result
+
+
+def test_verifier_app_virtual_memory(workspace):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "app.py").write_text("import mmap; region=mmap.mmap(-1,512*1024**2); print(len(region))")
+    print("input_bytes=", len((repo / "app.py").read_bytes()))
+    demo = workspace / "demo.json"
+    demo.write_text(json.dumps(spec()["demo"]))
+    result = verify_side_task(Side(repo), demo, "path")
+    assert result.outcome == "observed", result

tests/test_search_verify.py

@@ -33,7 +33,9 @@ def test_real_differential(tmp_path, family):
             pytest.skip("host policy prohibits localhost sockets")
     sides, demos = materialise(tmp_path / family, family)
     assert (sides[0].checkout / "app.py").stat().st_size > 0
-    result = verify(*sides, demos["real"], family)
+    result = verify(
+        *sides, demos["real"], {"sql": "injection", "command": "injection", "ssrf": "untrusted_destination"}.get(family, family)
+    )
     assert result.outcome == "demonstrated", result
     assert [r.observed for r in result.sides[0].runs] == [True] * 3
     assert [r.observed for r in result.sides[1].runs] == [False] * 3
@@ -79,7 +81,7 @@ def test_readiness_failure(pair):
     assert result.sides[1].outcome == "could_not_run"
 
 
-@pytest.mark.parametrize("family", ["xss", "deserialisation", "access_control", "secrets"])
+@pytest.mark.parametrize("family", ["deserialization", "access_control", "config_secrets"])
 def test_no_oracle(pair, family):
     sides, demo = pair
     assert verify(*sides, demo, family).outcome == "no_oracle"
@@ -472,3 +474,43 @@ def test_http_failure_keeps_exception_tail(tmp_path):
             lambda *a, **kw: SandboxResult(1, "", traceback, False),
             lambda: None,
         )
+
+
+@pytest.mark.parametrize(
+    "family,oracle",
+    [
+        ("injection", "sql"),
+        ("injection", "command"),
+        ("path", "path"),
+        ("output_encoding", "xss"),
+        ("untrusted_destination", "ssrf"),
+    ],
+)
+def test_verifier_routes_demo_oracle(pair, monkeypatch, family, oracle):
+    from openultrasast.search import verify as module
+    from openultrasast.search.oracles import BrowserExecutor
+
+    sides, demo = pair
+    edit(demo, lambda value: value.update(oracle=oracle))
+    seen = []
+    monkeypatch.setattr(_sandbox, "isolation_check", lambda **kwargs: None)
+
+    def side(checkout, artefact, kind, *args):
+        seen.append(kind)
+        return SideRecord("could_not_run", (), 0, "stub")
+
+    monkeypatch.setattr(module, "_side", side)
+    result = verify(*sides, demo, family, oracle=oracle, browser=BrowserExecutor() if oracle == "xss" else None)
+    assert seen == [oracle, oracle] and result.outcome != "no_oracle"
+    edit(demo, lambda value: value.update(oracle="path" if oracle != "path" else "sql"))
+    seen.clear()
+    result = verify(*sides, demo, family)
+    assert result.outcome == "inconclusive" and not seen
+    assert "not allowed" in result.reason
+
+
+def test_xss_without_browser_is_unavailable_not_no_oracle(pair):
+    sides, demo = pair
+    edit(demo, lambda value: value.update(oracle="xss"))
+    result = verify(*sides, demo, "output_encoding")
+    assert result.outcome == "inconclusive" and "runtime unavailable" in result.reason

tests/test_search_verify_transport.py

@@ -17,6 +17,7 @@ from openultrasast.search.verify import RunObservation, Side, SideRecord
 
 def demo():
     return {
+        "oracle": "sql",
         "build": {"recipe": "none", "arguments": []},
         "start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
         "steps": [{"type": "cli", "arguments": []}],
@@ -43,7 +44,7 @@ def test_dispatch_uses_fresh_tasks_and_only_data(tmp_path):
     dispatch = transport.AXSideDispatcher(
         SimpleNamespace(), "image@sha256:" + "a" * 64, tmp_path / "out", lane=lane, prepare=lambda side, spec: b"executor-built-archive"
     )
-    result = dispatch(side=Side(root), demo=demo(), family="sql", timeout_seconds=2, fresh_task=True)
+    result = dispatch(side=Side(root), demo=demo(), family="injection", timeout_seconds=2, fresh_task=True)
     assert result.isolation_mode == "task-boundary"
     assert len(result.runs) == 3
     assert len(calls) == 3 and calls[0].id != calls[1].id
@@ -74,7 +75,7 @@ def test_entry_reads_input_and_scrubs_environment(tmp_path, monkeypatch, capsys)
 
     shutil.copytree(root, bundle / "checkout")
     (bundle / "products").mkdir()
-    (bundle / "spec.json").write_text(json.dumps({"demo": demo(), "family": "sql", "timeout_seconds": 2}))
+    (bundle / "spec.json").write_text(json.dumps({"demo": demo(), "family": "injection", "timeout_seconds": 2}))
     blob = entry.pack_checkout(bundle)
     monkeypatch.setattr(entry, "download", lambda url, limit: blob)
     outputs = []
@@ -151,7 +152,7 @@ def test_production_item_targets_packaged_module(tmp_path):
         dispatcher = transport.AXSideDispatcher(
             SimpleNamespace(), "image@sha256:" + "a" * 64, tmp_path / "out", prepare=lambda side, spec: b"built"
         )
-        dispatcher(side=Side(root), demo=demo(), family="sql", timeout_seconds=2, fresh_task=True)
+        dispatcher(side=Side(root), demo=demo(), family="injection", timeout_seconds=2, fresh_task=True)
     assert captured[0].url_env == frozenset({"VERIFY_INPUT_URL", "RESULT_URL"})
 
 
@@ -198,7 +199,7 @@ def test_default_dispatch_builds_in_executor_before_verifier(tmp_path, monkeypat
 
     monkeypatch.setattr(transport, "SearchExecutorTask", Executor)
     dispatch = transport.AXSideDispatcher(SimpleNamespace(), "image@sha256:" + "a" * 64, tmp_path / "out", lane=lane)
-    assert len(dispatch(side=Side(root), demo=demo(), family="sql", timeout_seconds=2, fresh_task=True).runs) == 3
+    assert len(dispatch(side=Side(root), demo=demo(), family="injection", timeout_seconds=2, fresh_task=True).runs) == 3
     assert events[3:] == ["verify"] * 3
 
 
@@ -214,3 +215,23 @@ def test_source_archive_omits_git_metadata(tmp_path):
     unpack_checkout(archive, target)
     assert (target / "app.py").read_text() == "print('source')"
     assert not (target / ".git").exists()
+
+
+@pytest.mark.parametrize(
+    "family,oracle",
+    [
+        ("injection", "sql"),
+        ("injection", "command"),
+        ("path", "path"),
+        ("output_encoding", "xss"),
+        ("untrusted_destination", "ssrf"),
+    ],
+)
+def test_transport_checks_family_and_demo_oracle(family, oracle):
+    schema = demo()
+    schema["oracle"] = oracle
+    spec = {"demo": schema, "family": family, "timeout_seconds": 2}
+    assert entry.validate_spec(spec) == spec
+    schema["oracle"] = "path" if oracle != "path" else "sql"
+    with pytest.raises(ValueError, match="not allowed"):
+        entry.validate_spec(spec)

tests/test_search_worker.py

@@ -85,7 +85,8 @@ def test_coordinator_stub_smoke_real_verify(tmp_path):
     def executor(job):
         if job.step != "verify":
             return worker(job)
-        result = verify(*sides, demo, "path")
+        assert job.payload["oracle"] == "path"
+        result = verify(*sides, demo, "path", oracle=job.payload["oracle"])
         outcomes.append(result.outcome)
         return {"status": "ok", "outcome": result.outcome, "evidence_refs": ["verify:local"], "cost_usd": 0}
 
@@ -198,6 +199,7 @@ def test_brain_refuses_repository_tools_without_executor(tmp_path):
 def test_write_demo_accepts_only_declarative_json(tmp_path):
     worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo")
     schema = {
+        "oracle": "path",
         "build": {"recipe": "none", "arguments": []},
         "start": {"runtime": "python", "path": "app.py", "arguments": [], "mode": "cli"},
         "steps": [{"type": "cli", "arguments": ["hello"]}],
@@ -386,3 +388,58 @@ def test_worker_exhaustion_is_structured(tmp_path, monkeypatch, detail):
     result = worker(task("explore"))
     assert result["budget_exhaustion"]["end_detail"] == detail
     assert result["end_detail"] == detail
+
+
+def malformed_tool(arguments="{"):
+    response = tool("read_file")
+    response["tool_calls"][0]["function"]["arguments"] = arguments
+    return response
+
+
+def test_malformed_arguments_recover_with_tool_result(tmp_path):
+    (tmp_path / "input").write_text("readable")
+    model = StubModel(
+        [
+            malformed_tool(),
+            tool("read_file", path="input"),
+            tool("finish", fact={"text": "read input", "evidence_refs": ["tool:1"]}),
+        ]
+    )
+    result = Worker(model, repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path))(task("explore"))
+    assert result["status"] == "ok" and result["fact"]["text"] == "read input"
+    feedback = model.calls[1]["messages"][-1]
+    assert feedback["role"] == "tool" and feedback["tool_call_id"] == "call-1"
+    assert json.loads(feedback["content"])["error"].startswith("invalid JSON arguments: ")
+    assert len(model.calls) == 3
+
+
+def test_two_malformed_arguments_record_failure(tmp_path):
+    model = StubModel([malformed_tool(), malformed_tool()])
+    result = Worker(model, repo=tmp_path, demo=tmp_path / "demo")(task("explore"))
+    assert result["status"] == "execution_failure"
+    assert "two consecutive malformed tool calls" in result["failure"]
+    assert len(model.calls) == 2
+
+
+def test_valid_call_resets_malformed_streak(tmp_path):
+    model = StubModel([malformed_tool(), tool("list_files"), malformed_tool(), tool("finish", failure="done")])
+    result = Worker(model, repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path))(task("explore"))
+    assert result["failure"] == "done" and len(model.calls) == 4
+
+
+def test_malformed_call_consumes_step(tmp_path):
+    model = StubModel([malformed_tool(), tool("finish", failure="unused")])
+    result = Worker(model, repo=tmp_path, demo=tmp_path / "demo", max_steps=1)(task("explore"))
+    assert result["end_detail"] == "worker_steps" and len(model.calls) == 1
+
+
+def test_worker_run_uses_search_memory_budget(tmp_path):
+    class Recorder:
+        def submit(self, name, args, **kwargs):
+            return kwargs["limits"]
+
+    worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo", executor=Recorder())
+    assert worker.tool("run", {"command": ["node", "app.js"]})["memory_bytes"] == SearchBudget().memory_bytes
+    assert worker.tool("grep", {"pattern": "needle"})["memory_bytes"] == 256 * 1024**2
+    worker.limits["memory_bytes"] = 128 * 1024**2
+    assert worker.tool("read_file", {"path": "app.py"})["memory_bytes"] == 128 * 1024**2