From 3866e2771b68c2c83e8f7675713e629fb221140f Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:37:56 +0200 Subject: [PATCH] search-with-proof: ecosystem-aware build recipes and stderr in build failures The agent passes a manifest file or project directory; pip uses -r for requirements files and the directory for pyproject/setup, npm ci/install runs in the directory, composer/maven/gradle likewise. Build and verify failures carry phase, exit code and a sanitised stderr tail. Co-Authored-By: Claude Fable 5.1 --- benchmarks/ax/batch.py | 6 +- benchmarks/ax/search_verify.py | 3 + benchmarks/search/pilot_run.py | 14 ++- src/openultrasast/search/demo.py | 76 +++++++++++++- src/openultrasast/search/executor.py | 36 +++---- src/openultrasast/search/probe.py | 101 +++++++++++++++--- src/openultrasast/search/task_storage.py | 17 ++- src/openultrasast/search/verify.py | 44 +++++--- src/openultrasast/search/worker.py | 7 +- tests/test_ax_batch.py | 24 +++++ tests/test_search_demo.py | 18 +++- tests/test_search_pilot_run.py | 20 ++++ tests/test_search_probe.py | 128 +++++++++++++++++++++++ tests/test_search_task_storage.py | 109 ++++++++++++++++++- 14 files changed, 543 insertions(+), 60 deletions(-) diff --git a/benchmarks/ax/batch.py b/benchmarks/ax/batch.py index ecf4fcd..a2268e4 100644 --- a/benchmarks/ax/batch.py +++ b/benchmarks/ax/batch.py @@ -724,7 +724,11 @@ class SearchExecutorTask: remaining = min(300, self.end - time.monotonic()) result = self.client.submit("prepare_verification", spec, timeout_seconds=remaining) if result.get("status") != "ok": - raise ValueError(result.get("reason", "could_not_build")) + from openultrasast.search.task_storage import CommandFailure + + raise CommandFailure( + result.get("phase", "build"), result.get("exit_code"), result.get("stderr", ""), result.get("reason", "could_not_build") + ) stored = bounded_read(lambda: self.lane.store._get(self.keys[3]), self.end - time.monotonic()) if stored is None or hashlib.sha256(stored[0]).hexdigest() != result.get("sha256"): raise ValueError("executor archive missing or digest mismatch") diff --git a/benchmarks/ax/search_verify.py b/benchmarks/ax/search_verify.py index 6b810a6..082ad00 100644 --- a/benchmarks/ax/search_verify.py +++ b/benchmarks/ax/search_verify.py @@ -110,4 +110,7 @@ class AXSideDispatcher: max(r.scratch_peak_bytes for r in records), isolation_mode="task-boundary", chromium_no_sandbox=any(r.chromium_no_sandbox for r in records), + phase=last.phase, + exit_code=last.exit_code, + stderr=last.stderr, ) diff --git a/benchmarks/search/pilot_run.py b/benchmarks/search/pilot_run.py index e677ffb..d604ed5 100644 --- a/benchmarks/search/pilot_run.py +++ b/benchmarks/search/pilot_run.py @@ -123,7 +123,7 @@ class Pilot: self.sides = None self.verifications = [] - def failure(self, phase, reason): + def failure(self, phase, reason, command=None): private = tuple(str(v) for k, v in self.row.items() if k in {"repo", "file", "function", "vulnerable", "fixed"}) private += (str(self.root), self.row["repo"].removeprefix("https://github.com/")) if isinstance(reason, BaseException): @@ -135,7 +135,15 @@ class Pilot: safe = kind + ": " + diagnostic(message, maximum=max(0, 298 - len(kind)), private=private) else: safe = diagnostic(reason, private=private) - self.record["failures"].append(dict(phase=phase, reason=safe)) + entry = dict(phase=phase, reason=safe) + command = command if command is not None else reason + if hasattr(command, "exit_code"): + entry.update( + command_phase=diagnostic(command.phase), + exit_code=command.exit_code, + stderr=diagnostic(command.stderr, maximum=1500, private=private), + ) + self.record["failures"].append(entry) @contextmanager def executor(self, revision, deadline=900): @@ -300,7 +308,7 @@ class Pilot: for side in result.sides: if side.outcome != "observed": - self.failure("verify/" + side.outcome, "side_verification_failed") + self.failure("verify/" + side.outcome, side.reason, command=side) if result.outcome != "demonstrated": self.failure("verify", "no_consistent_differential" if result.outcome == "inconclusive" else result.outcome) end = result.outcome diff --git a/src/openultrasast/search/demo.py b/src/openultrasast/search/demo.py index e52d799..1fbf342 100644 --- a/src/openultrasast/search/demo.py +++ b/src/openultrasast/search/demo.py @@ -9,6 +9,7 @@ from __future__ import annotations import json import re +import tomllib from pathlib import Path, PurePosixPath from typing import Any, cast @@ -145,6 +146,75 @@ def checkout_path(value: Any) -> str: return value +_MANIFESTS = { + "pip": ("pyproject.toml", "setup.py", "setup.cfg"), + "npm": ("package.json", "package-lock.json"), + "composer": ("composer.json", "composer.lock"), + "maven": ("pom.xml",), + "gradle": ("build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts", "gradlew"), +} + + +def build_project(recipe: str, path: Path, checkout: Path) -> Path: + """Validate the recipe ecosystem against actual checkout files.""" + + def manifest(candidate: Path) -> bool: + return candidate.name in _MANIFESTS[recipe] or (recipe == "pip" and candidate.match("requirements*.txt")) + + def contained(candidate: Path) -> bool: + return candidate.resolve().is_relative_to(checkout.resolve()) + + if not contained(path): + raise ValueError("recipe input outside checkout") + if path.is_file(): + if not manifest(path): + raise ValueError("recipe input is not an ecosystem manifest") + return path.parent + if path.is_dir() and any(p.is_file() and contained(p) and manifest(p) for p in path.iterdir()): + return path + raise ValueError("recipe input must be an ecosystem manifest or project directory") + + +def preparation_command(recipe: str, path: Path, checkout: Path, products: Path) -> tuple[list[str], Path]: + project = build_project(recipe, path, checkout) + if recipe == "pip": + requirements = path if path.is_file() and path.match("requirements*.txt") else None + pyproject = project / "pyproject.toml" + backend = False + if requirements is None and pyproject.is_file(): + if not pyproject.resolve().is_relative_to(checkout.resolve()): + raise ValueError("recipe input outside checkout") + backend = bool(tomllib.loads(pyproject.read_text()).get("build-system", {}).get("build-backend")) + if requirements is None and not backend and not (project / "setup.py").is_file(): + fallback = project / "requirements.txt" + if fallback.is_file(): + requirements = fallback + target = requirements or project + if not target.resolve().is_relative_to(checkout.resolve()): + raise ValueError("recipe input outside checkout") + return [ + "python3", + "-I", + "-m", + "pip", + "install", + "--no-deps", + "--target", + str(products / "packages"), + *(["-r", str(target)] if requirements else [str(target)]), + ], project + if recipe == "npm": + return ["npm", "ci" if (project / "package-lock.json").is_file() else "install", "--ignore-scripts"], project + if recipe == "composer": + return ["composer", "install", "--no-interaction", "--no-plugins", "--no-scripts"], project + if recipe == "maven": + return ["mvn", "-q", "-DskipTests", "package", "-f", str(project / "pom.xml")], project + wrapper = project / "gradlew" + if not wrapper.is_file() or not wrapper.resolve().is_relative_to(checkout.resolve()): + raise ValueError("missing or external Gradle wrapper") + return ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(project), "assemble"], project + + def _text(value: Any, names: set[str]) -> str: if not isinstance(value, str) or "\0" in value or len(value.encode()) > MAX_VALUE_BYTES: raise ValueError("invalid or oversized value") @@ -163,7 +233,7 @@ def _arguments(value: Any, names: set[str]) -> None: _text(arg, names) -def validate_demo(value: Any) -> dict[str, Any]: +def validate_demo(value: Any, *, checkout: Path | None = None) -> dict[str, Any]: try: encoded = json.dumps(value, allow_nan=False).encode() except (ValueError, TypeError, RecursionError) as exc: @@ -186,8 +256,10 @@ def validate_demo(value: Any) -> dict[str, Any]: else: if len(args) != 1: raise ValueError("recipe requires one checkout path") - if not (recipe in ("npm", "composer", "gradle") and args[0] == "."): + if args[0] != ".": checkout_path(args[0]) + if checkout is not None: + build_project(recipe, checkout / args[0], checkout) _keys(start, {"runtime", "path", "arguments", "mode"}, {"port"}) if not isinstance(start["runtime"], str) or start["runtime"] not in RUNTIMES: raise ValueError("unknown runtime") diff --git a/src/openultrasast/search/executor.py b/src/openultrasast/search/executor.py index a0c1a42..ae3d3f9 100644 --- a/src/openultrasast/search/executor.py +++ b/src/openultrasast/search/executor.py @@ -269,12 +269,12 @@ class InProcessExecutor: "isolation_mode": self.isolation_mode, "exit_code": result.exit_code, "stdout": result.stdout[:16384], - "stderr": task_storage.diagnostic(result.stderr, maximum=4000), + "stderr": task_storage.diagnostic(result.stderr, maximum=1500), "timed_out": result.timed_out, "truncated": len(result.stdout) > 16384 or len(result.stderr) > 16384, } - def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]: + def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any], *, cwd: Path | None = None) -> dict[str, Any]: process_limit = None try: uid = os.getuid() @@ -311,7 +311,7 @@ class InProcessExecutor: self.phase = "spawn" with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err: proc = subprocess.Popen( - argv, cwd=self.repo, env=clean_environment(), stdout=out, stderr=err, start_new_session=True, preexec_fn=constrain + argv, cwd=cwd or self.repo, env=clean_environment(), stdout=out, stderr=err, start_new_session=True, preexec_fn=constrain ) self.phase = "run" timed_out = False @@ -358,7 +358,7 @@ class InProcessExecutor: proc.wait() out.seek(0) err.seek(max(0, os.fstat(err.fileno()).st_size - 16384)) - stderr = task_storage.diagnostic(err.read().decode(errors="replace"), maximum=4000) + stderr = task_storage.diagnostic(err.read().decode(errors="replace"), maximum=1500) phase = "scratch guard" if disk_exceeded else "timeout" if timed_out else self.phase if disk_exceeded: self.stopped = True @@ -389,14 +389,14 @@ class InProcessExecutor: """Build and export through the host-scoped output URL, never a model URL.""" import shutil - from .demo import BUILD_RECIPES, validate_demo + from .demo import preparation_command, validate_demo from .verify import _tree from .verify_task import pack_checkout, validate_spec if not self.task_boundary or not self.prepared_put_url: raise ValueError("preparation requires an executor task output") spec = copy.deepcopy(spec) - validate_demo(spec["demo"]) + validate_demo(spec["demo"], checkout=self.repo) recipe = spec["demo"]["build"] spec["demo"]["build"] = {"recipe": "none", "arguments": []} validate_spec(spec) @@ -406,22 +406,16 @@ class InProcessExecutor: products.mkdir() if recipe["recipe"] != "none": path = self._path(recipe["arguments"][0]) - argv = [ - arg.replace("/scratch", str(products)) - for arg in BUILD_RECIPES[recipe["recipe"]] - if arg not in {"--offline", "--no-index"} - ] - argv.append(str(path)) - if recipe["recipe"] == "maven": - argv += ["package", "-DskipTests"] - elif recipe["recipe"] == "gradle": - # The project pins its Gradle version through its wrapper. - # Invoke with sh because checkout archives may lose execute bits. - wrapper = self._path(str(path.relative_to(self.repo) / "gradlew")) - argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"] - built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES, **limits}) + argv, project = preparation_command(recipe["recipe"], path, self.repo, products) + self.phase = "build" + built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES, **limits}, cwd=project) if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]: - return {**built, "status": "could_not_build"} + return { + **built, + "status": "could_not_build", + "phase": "build", + "stderr": task_storage.diagnostic(built.get("stderr", ""), maximum=1500), + } # Validate internal generated links before materializing them as # regular archive members. No host/external files may be exported. _tree(self.repo, generated_links=True) diff --git a/src/openultrasast/search/probe.py b/src/openultrasast/search/probe.py index 363f280..7167637 100644 --- a/src/openultrasast/search/probe.py +++ b/src/openultrasast/search/probe.py @@ -15,16 +15,16 @@ import subprocess import tempfile import time import urllib.request -from dataclasses import asdict +from dataclasses import asdict, replace from importlib.resources import files from pathlib import Path from typing import Any from . import _sandbox -from .demo import FAMILY_ORACLES +from .demo import FAMILY_ORACLES, load_demo from .oracles import BrowserExecutor from .task_storage import exception_reason -from .verify import Side, verify +from .verify import Side, VerificationRecord, verify, verify_side_task FAMILIES = ("sql", "path", "command", "ssrf", "xss") PAYLOADS = { @@ -105,6 +105,55 @@ def namespace_facts() -> dict[str, Any]: return facts +def _verify_task_pair(sides: tuple[Side, Side], demo: Path, family: str) -> VerificationRecord: + """Exercise the task executor on packaged toys only, not arbitrary checkouts. + + Repetitions get fresh scratch, but share this probe task's outer boundary. + This is an instrument self-test, not a fresh-task dispatcher for evidence. + """ + started = time.monotonic() + records = [] + try: + load_demo(demo) + except (OSError, ValueError) as exc: + return VerificationRecord( + "inconclusive", + (), + time.monotonic() - started, + "invalid declarative demo: " + exception_reason(exc), + isolation_mode="task-boundary", + ) + search_family = next(f for f, kinds in FAMILY_ORACLES.items() if family in kinds) + for side in sides: + runs = [verify_side_task(side, demo, search_family) for _ in range(3)] + first = next((run for run in runs if run.outcome != "observed"), runs[0]) + records.append( + replace( + first, + runs=tuple(observation for run in runs for observation in run.runs), + elapsed_seconds=sum(run.elapsed_seconds for run in runs), + build_seconds=sum(run.build_seconds for run in runs), + ready_seconds=sum(run.ready_seconds for run in runs), + run_seconds=sum(run.run_seconds for run in runs), + scratch_peak_bytes=max(run.scratch_peak_bytes for run in runs), + reason=first.reason if first.outcome != "observed" else "three probe task executions completed", + ) + ) + demonstrated = ( + all(side.outcome == "observed" and len(side.runs) == 3 for side in records) + and all(run.observed for run in records[0].runs) + and not any(run.observed for run in records[1].runs) + ) + return VerificationRecord( + "demonstrated" if demonstrated else "inconclusive", + tuple(records), + time.monotonic() - started, + "owned effect in 3/3 affected runs and 0/3 safe runs" if demonstrated else "no consistent differential effect", + isolation_mode="task-boundary", + chromium_no_sandbox=any(side.chromium_no_sandbox for side in records), + ) + + def probe(root: Path) -> dict[str, Any]: record: dict[str, Any] = { "status": "ok", @@ -114,26 +163,54 @@ def probe(root: Path) -> dict[str, Any]: "isolation_check": None, "isolation_mode": _sandbox.isolation_mode(), } + namespaces = record["namespaces"] + if namespaces and all(namespaces[name]["exit_code"] != 0 for name in ("user", "pid", "net", "ipc", "uts", "mount")): + record["isolation_mode"] = "task-boundary" + task_boundary = record["isolation_mode"] == "task-boundary" + if task_boundary: + # Real verification still needs fresh side tasks; only these packaged + # self-test applications share the probe task's boundary. + record["requires_fresh_side_tasks"] = True start = time.monotonic() try: - _sandbox.isolation_check() - record["isolation_check"] = {"status": "ok", "input_bytes": len(b"isolation-ready\n")} + if task_boundary: + record["isolation_check"] = {"status": "skipped", "reason": "task-boundary"} + else: + _sandbox.isolation_check() + record["isolation_check"] = {"status": "ok", "input_bytes": len(b"isolation-ready\n")} for family in FAMILIES: sides, demos = materialise(root / family, family) results: dict[str, Any] = {} record["oracles"][family] = results for name, demo in demos.items(): - result = verify( - *sides, - demo, - next(f for f, kinds in FAMILY_ORACLES.items() if family in kinds), - browser=BrowserExecutor() if family == "xss" else None, - ) + search_family = next(f for f, kinds in FAMILY_ORACLES.items() if family in kinds) + if task_boundary: + tick = time.monotonic() + try: + result = _verify_task_pair(sides, demo, family) + except Exception as exc: + result = VerificationRecord( + "inconclusive", (), time.monotonic() - tick, exception_reason(exc), isolation_mode="task-boundary" + ) + else: + result = verify( + *sides, + demo, + search_family, + oracle=family, + browser=BrowserExecutor() if family == "xss" else None, + ) results[name] = asdict(result) + if task_boundary and (result.outcome == "demonstrated") != (name == "real") and record["status"] == "ok": + expected = "demonstrated" if name == "real" else "not demonstrated" + record.update( + status="instrument_failure", + exception=f"{family}/{name}: expected {expected}, got {result.outcome}", + ) # SSRF/browser need additional namespace capabilities beyond the # generic preflight. These failures must not look like oracle misses. for side in result.sides: - if side.reason.startswith("verification unavailable or refused:"): + if not task_boundary and side.reason.startswith("verification unavailable or refused:"): raise _sandbox.IsolationUnavailable(f"{family}/{name}: {side.reason}") except Exception as exc: record.update(status="instrument_failure", exception=exception_reason(exc)) diff --git a/src/openultrasast/search/task_storage.py b/src/openultrasast/search/task_storage.py index 456d5f4..1773c9b 100644 --- a/src/openultrasast/search/task_storage.py +++ b/src/openultrasast/search/task_storage.py @@ -106,8 +106,12 @@ def diagnostic(value: object, *, maximum: int = 300, private: tuple[str, ...] = text = re.sub(r"(?i)bearer\s+[^\s,;\"']+", "Bearer [redacted]", text) text = re.sub(r"(?i)(?:https?|s3)://[^\s<>\"']+", "[redacted-url]", text) text = re.sub( - r"(?i)([\w-]*(?:authorization|signature|credential|token|secret|password|key))\s*[:=]\s*[^\s,;&\"']+", r"\1=[redacted]", text + r"(?i)([\w-]*(?:authorization|signature|credential|token|secret|password|key))[\"']?\s*[:=]\s*[\"']?[^\s,;&\"']+", + r"\1=[redacted]", + text, ) + text = re.sub(r"\b(?:gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+|sk-[A-Za-z0-9_-]+|npm_[A-Za-z0-9]+)\b", "[redacted]", text) + text = re.sub(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b", "[redacted]", text) for name in sorted(set(private), key=len, reverse=True): if name: text = text.replace(name, "[repository]") @@ -125,3 +129,14 @@ def exception_reason(exc: BaseException, *, private: tuple[str, ...] = ()) -> st half = max(0, (maximum - 2) // 2) return prefix + diagnostic(original, maximum=half, private=private) + "; " + diagnostic(cleanup, maximum=half, private=private) return prefix + diagnostic(exc, maximum=maximum, private=private) + + +class CommandFailure(ValueError): + """Transport a bounded command diagnostic without dropping stderr.""" + + def __init__(self, phase: str, exit_code: int | None, stderr: str, reason: str = "") -> None: + self.phase = phase + self.exit_code = exit_code + self.stderr = diagnostic(stderr, maximum=1500) + summary = diagnostic(reason) if reason else f"{phase}: exit code {exit_code}" + super().__init__(summary + (": " + self.stderr if self.stderr else "")) diff --git a/src/openultrasast/search/verify.py b/src/openultrasast/search/verify.py index 6946255..cebacf1 100644 --- a/src/openultrasast/search/verify.py +++ b/src/openultrasast/search/verify.py @@ -30,7 +30,7 @@ from . import _sandbox, task_storage from .budget import SearchBudget from .demo import BUILD_RECIPES, FAMILY_ORACLES, RUNTIMES, capture_output, load_demo, substitute, validate_oracle from .oracles import BrowserExecutor, Canary, Oracle, oracle_for -from .task_storage import exception_reason +from .task_storage import CommandFailure, diagnostic, exception_reason @dataclass(frozen=True) @@ -62,6 +62,9 @@ class SideRecord: scratch_peak_bytes: int = 0 isolation_mode: str = field(default_factory=_sandbox.isolation_mode) chromium_no_sandbox: bool = False + phase: str = "" + exit_code: int | None = None + stderr: str = "" @dataclass(frozen=True) @@ -133,18 +136,19 @@ def _tree(root: Path, *, generated_links: bool = False) -> str: # Executed as verifier-owned code inside the same network sandbox as the app. # Redirects are refused: only the configured loopback application is reachable. _HTTP_DRIVER = r""" -import http.client, importlib.util, json, os, socket, subprocess, sys, time +import http.client, importlib.util, json, os, socket, subprocess, sys, tempfile, time spec = importlib.util.spec_from_file_location('demo', '/verifier-demo.py') demo = importlib.util.module_from_spec(spec) spec.loader.exec_module(demo) config = json.load(sys.stdin) schema = config['schema'] -app = subprocess.Popen(config['command'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +app_errors = tempfile.TemporaryFile() +app = subprocess.Popen(config['command'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=app_errors) try: until = time.monotonic() + config['timeout'] / 2 while True: if app.poll() is not None: - raise RuntimeError('HTTP application exited during readiness') + raise SystemExit(app.returncode or 1) try: connection = socket.create_connection(('127.0.0.1', schema['start']['port']), timeout=0.1) connection.close() @@ -183,6 +187,9 @@ finally: except subprocess.TimeoutExpired: app.kill() app.wait() + app_errors.seek(max(0, app_errors.seek(0, 2) - 16384)) + sys.stderr.write(app_errors.read().decode(errors="replace")) + app_errors.close() """ @@ -223,7 +230,12 @@ def _http_steps( acknowledged = True result = future.result() if not acknowledged or result.exit_code or result.timed_out: - raise ValueError("HTTP application/steps failed or readiness not acknowledged: " + result.stderr[-2000:]) + raise CommandFailure( + "start" if not acknowledged else "run", + result.exit_code, + result.stderr, + "HTTP application/steps failed or readiness not acknowledged", + ) outputs = json.loads(result.stdout) if not isinstance(outputs, list) or len(outputs) != len(schema["steps"]) or not all(isinstance(value, str) for value in outputs): raise ValueError("invalid HTTP driver output") @@ -249,18 +261,21 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe if root is not None: disk_peak = max(disk_peak, sum(p.lstat().st_blocks * 512 for p in root.rglob("*"))) - def record(outcome: str, reason: str) -> SideRecord: + def record(outcome: str, reason: str, phase: str = "", result: Any = None) -> SideRecord: return SideRecord( outcome, tuple(observations), time.monotonic() - started, - reason, + diagnostic(reason, maximum=1500), timings["build"], timings["ready"], timings["run"], disk_peak, isolation_mode="task-boundary" if task_boundary else _sandbox.isolation_mode(), chromium_no_sandbox=bool(getattr(browser, "no_sandbox", False)), + phase=phase, + exit_code=result.exit_code if result is not None else None, + stderr=diagnostic(result.stderr, maximum=1500) if result is not None else "", ) try: @@ -305,7 +320,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe command += ("assemble",) compiled = timed("build", runner, job(command), scratch=products, mounts=mounts, writable_checkout=True) if compiled.exit_code or compiled.timed_out: - return record("could_not_build", "build recipe failed: " + compiled.stderr[-2000:]) + return record("could_not_build", "build recipe failed", "build", compiled) if tuple(_tree(p) for p in immutable) != before: return record("inconclusive", "build changed immutable inputs") product_hash = _tree(products, generated_links=True) @@ -354,7 +369,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe namespace_pid=oracle.namespace_pid, ) if ready.exit_code or ready.timed_out: - return record("could_not_run", "app readiness failed: " + ready.stderr[-2000:]) + return record("could_not_run", "app readiness failed", "start", ready) if family == "ssrf" and oracle.observe()[0]: return record("inconclusive", "readiness triggered the SSRF oracle") # Readiness cannot supply the attack's evidence or leave a command marker. @@ -401,7 +416,7 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe if result.timed_out: return record("inconclusive", "app attack timed out") if result.exit_code: - return record("could_not_run", "app invocation failed") + return record("could_not_run", "app invocation failed", "run", result) timed("run", oracle.capture, result.stdout) capture_output(step, result.stdout, captures, canary.nonce) observed, evidence = timed("run", oracle.observe) @@ -416,6 +431,8 @@ def _side(side: Side, demo: Path, family: str, timeout: int, browser: BrowserExe finally: oracle.close() return record("observed", "fresh task run completed" if task_boundary else "three fresh runs completed") + except CommandFailure as exc: + return record("could_not_run", str(exc), exc.phase, exc) except task_storage.ScratchLimit as exc: return record("could_not_build", exception_reason(exc)) except (OSError, ValueError, AssertionError, KeyError, IndexError, TypeError) as exc: @@ -437,10 +454,11 @@ def _run_task( the filesystem/network boundary. Its launch manifest must contain no keys. """ translations = {"/workspace": str(job.repo_root), "/scratch": str(scratch), **{k: str(v) for k, v in (mounts or {}).items()}} - pattern = re.compile("|".join(re.escape(key) for key in sorted(translations, key=len, reverse=True))) + pattern = re.compile("(?:" + "|".join(re.escape(key) for key in sorted(translations, key=len, reverse=True)) + r")(?=/|$|['\"])") def translate(value: str) -> str: - # One substitution pass: host temp paths may themselves contain /scratch. + # One pass, at path-component boundaries: /fixture/fixture.db must + # translate its mount prefix without also rewriting the filename. return pattern.sub(lambda match: translations[match.group(0)], value) command = tuple(translate(arg) for arg in job.command) @@ -471,7 +489,7 @@ def _run_task( os.killpg(proc.pid, signal.SIGKILL) proc.wait() out.seek(0) - err.seek(0) + err.seek(max(0, os.fstat(err.fileno()).st_size - 16384)) return SandboxResult( proc.returncode, out.read(1048576).decode(errors="replace"), err.read(1048576).decode(errors="replace"), expired ) diff --git a/src/openultrasast/search/worker.py b/src/openultrasast/search/worker.py index b51fd94..ce6cd5d 100644 --- a/src/openultrasast/search/worker.py +++ b/src/openultrasast/search/worker.py @@ -81,7 +81,12 @@ DEMO_GUIDE = ( 'Demo schema: {"oracle":"path","build":{"recipe":"none","arguments":[]},' '"start":{"runtime":"python","path":"app.py","arguments":[],"mode":"cli"},' '"steps":[{"type":"cli","arguments":["hello"]}]}. This is a minimal example, not evidence. ' - "Recipes: none,pip,npm,composer,maven,gradle; none takes [], others one checkout-relative manifest/project path. " + "Recipes: none takes [], otherwise arguments[0] is a checkout-relative ecosystem manifest or project directory (a file uses " + "its parent): pip requirements*.txt uses install -r (e.g. requirements.txt), pyproject.toml/setup.py/setup.cfg uses " + "non-editable install of the directory with requirements.txt fallback when no build backend exists (e.g. pyproject.toml); npm uses " + "ci with package-lock.json else install in the directory (e.g. package.json); composer uses install --no-interaction in the " + "directory (e.g. composer.json); maven uses mvn -q -DskipTests package -f (e.g. pom.xml); gradle uses the project " + "wrapper and directory (e.g. build.gradle). " "Runtimes: python,node,php,java; start.path must be a tracked checkout entrypoint. " "HTTP mode requires port (1024-65535) and steps {type:http,method,path,headers?,body?}; " "CLI mode forbids port and uses {type:cli,arguments,stdin?}. Supply 1-32 ordered steps. " diff --git a/tests/test_ax_batch.py b/tests/test_ax_batch.py index 33505b9..a4f3f1a 100644 --- a/tests/test_ax_batch.py +++ b/tests/test_ax_batch.py @@ -385,3 +385,27 @@ def test_executor_slot_deadline_has_reason(monkeypatch): monkeypatch.setattr(batch, "AX_SLOTS", SimpleNamespace(acquire=lambda **kw: False)) with pytest.raises(batch.CapacityDeadline, match="no free workers available before deadline"): batch.SearchExecutorTask(object(), b"archive").__enter__() + + +def test_search_preparation_failure_retains_command_record(): + import time + from types import SimpleNamespace + + from benchmarks.ax.batch import SearchExecutorTask + from openultrasast.search.task_storage import CommandFailure + + task = object.__new__(SearchExecutorTask) + task.end = time.monotonic() + 60 + task.client = SimpleNamespace( + submit=lambda *args, **kwargs: { + "status": "could_not_build", + "phase": "build", + "exit_code": 8, + "reason": "CommandFailure: exit code 8", + "stderr": "dependency missing token=hidden", + } + ) + with pytest.raises(CommandFailure) as caught: + task.prepare({}) + assert caught.value.phase == "build" and caught.value.exit_code == 8 + assert "dependency missing" in caught.value.stderr and "hidden" not in caught.value.stderr diff --git a/tests/test_search_demo.py b/tests/test_search_demo.py index e5a0483..98c3bba 100644 --- a/tests/test_search_demo.py +++ b/tests/test_search_demo.py @@ -71,11 +71,10 @@ def test_directory_build_recipe_accepts_checkout_root(recipe): @pytest.mark.parametrize("recipe", ["pip", "maven"]) -def test_manifest_recipe_requires_file_path(recipe): +def test_manifest_recipe_accepts_project_directory(recipe): value = document() value["build"] = {"recipe": recipe, "arguments": ["."]} - with pytest.raises(ValueError): - validate_demo(value) + assert validate_demo(value) == value @pytest.mark.parametrize("oracle", [None, "injection", "unknown", [], 1]) @@ -111,3 +110,16 @@ def test_family_oracle_contract(family, allowed): else: with pytest.raises(ValueError, match="not allowed"): validate_oracle(family, oracle) + + +@pytest.mark.parametrize( + "recipe,manifest", + [("pip", "package.json"), ("npm", "requirements.txt"), ("composer", "pom.xml"), ("maven", "composer.json"), ("gradle", "setup.py")], +) +@pytest.mark.parametrize("directory", [False, True]) +def test_recipe_rejects_other_ecosystem(tmp_path, recipe, manifest, directory): + (tmp_path / manifest).write_text("{}") + value = document() + value["build"] = {"recipe": recipe, "arguments": ["." if directory else manifest]} + with pytest.raises(ValueError, match="ecosystem"): + validate_demo(value, checkout=tmp_path) diff --git a/tests/test_search_pilot_run.py b/tests/test_search_pilot_run.py index fca48db..bc8d97d 100644 --- a/tests/test_search_pilot_run.py +++ b/tests/test_search_pilot_run.py @@ -480,3 +480,23 @@ def test_injection_pilot_uses_demo_choice_without_manifest_subtype(tmp_path, ora (tmp_path / "demo/demo.json").write_bytes((demos["real"] / "demo.json").read_bytes()) result = pilot.verification() assert result.outcome == "demonstrated", result + + +@pytest.mark.parametrize("phase", ["build_export", "verify/could_not_run"]) +def test_pilot_retains_command_diagnostics(tmp_path, phase): + from openultrasast.search.task_storage import CommandFailure + from openultrasast.search.verify import SideRecord + + pilot = make_pilot(tmp_path) + stderr = "\n".join(f"line-{i}" for i in range(30)) + "\ntoken=hidden https://example.test/?key=hidden" + if phase == "build_export": + pilot.failure(phase, CommandFailure("build", 7, stderr)) + else: + side = SideRecord("could_not_run", (), 0, "start failed", phase="start", exit_code=7, stderr=stderr) + pilot.failure(phase, side.reason, command=side) + failure = pilot.record["failures"][-1] + assert failure["exit_code"] == 7 + assert failure["command_phase"] == ("build" if phase == "build_export" else "start") + assert "line-29" in failure["stderr"] and "line-0\n" not in failure["stderr"] + assert "hidden" not in failure["stderr"] and "https://" not in failure["stderr"] + assert len(failure["stderr"]) <= 1500 diff --git a/tests/test_search_probe.py b/tests/test_search_probe.py index 669b9e1..8f7126e 100644 --- a/tests/test_search_probe.py +++ b/tests/test_search_probe.py @@ -2,12 +2,22 @@ import json import sys from pathlib import Path +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from openultrasast.search import probe from openultrasast.search._sandbox import IsolationUnavailable +@pytest.fixture(autouse=True) +def namespace_diagnostics(monkeypatch): + facts = {name: {"exit_code": 0} for name in ("user", "pid", "net", "ipc", "uts", "mount")} + monkeypatch.setattr(probe, "namespace_facts", lambda: facts) + monkeypatch.setattr(probe._sandbox, "isolation_mode", lambda: "userns") + return facts + + def test_isolation_failure_is_recorded_and_nonzero(monkeypatch, capsys): def fail(**kwargs): raise IsolationUnavailable("gVisor denied namespace") @@ -118,3 +128,121 @@ def test_oracle_namespace_failure_is_instrument_failure(tmp_path, monkeypatch): assert record["status"] == "instrument_failure" assert "net namespace denied" in record["exception"] assert record["isolation_check"]["status"] == "ok" + + +@pytest.mark.parametrize("execution", ["real", "stubbed"]) +def test_task_boundary_probe_executes_packaged_pairs(tmp_path, monkeypatch, namespace_diagnostics, execution): + if execution == "real": + import socket + + try: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + except PermissionError: + pytest.skip("host policy prohibits localhost sockets required by SSRF and Chromium") + else: + from openultrasast.search.verify import RunObservation, SideRecord + + def run(side, demo, family): + assert (side.checkout / "app.py").read_bytes() + observed = not (side.checkout / "fixed").exists() + return SideRecord( + "observed", + (RunObservation(observed, "stubbed owned effect", 0.5),), + 0.7, + "stubbed execution", + 0, + 0.2, + 0.5, + 4096, + "task-boundary", + json.loads((demo / "demo.json").read_text())["oracle"] == "xss", + ) + + monkeypatch.setattr(probe, "verify_side_task", run) + monkeypatch.setattr(probe._sandbox, "isolation_mode", lambda: "task-boundary") + monkeypatch.setattr(probe, "image_facts", lambda: {}) + monkeypatch.setattr(probe._sandbox, "isolation_check", lambda **kw: pytest.fail("task boundary must skip preflight")) + monkeypatch.setattr(probe, "verify", lambda *a, **kw: pytest.fail("must use task executor")) + record = probe.probe(tmp_path) + assert record["status"] == "ok", record + assert record["isolation_check"] == {"status": "skipped", "reason": "task-boundary"} + assert record["isolation_mode"] == "task-boundary" + assert record["namespaces"] == namespace_diagnostics + assert set(record["oracles"]) == set(probe.FAMILIES) + for family, demos in record["oracles"].items(): + assert set(demos) == {"real", "observer_access", "revision_patch"} + for name, result in demos.items(): + assert (result["outcome"] == "demonstrated") == (name == "real") + assert result["isolation_mode"] == "task-boundary" + assert result["elapsed_seconds"] > 0 + real = demos["real"] + assert real["chromium_no_sandbox"] == (family == "xss") + for index, side in enumerate(real["sides"]): + assert len(side["runs"]) == 3 + assert all(run["observed"] == (index == 0) for run in side["runs"]) + assert side["isolation_mode"] == "task-boundary" + assert side["chromium_no_sandbox"] == (family == "xss") + assert side["build_seconds"] == 0 + assert side["ready_seconds"] > 0 + assert side["run_seconds"] > 0 + assert side["scratch_peak_bytes"] > 0 + assert record["wall_seconds"] > 0 + assert record["peak_rss_kib"]["sum"] > 0 + assert record["scratch_disk_bytes"] > 0 + assert record["verification_scratch_peak_bytes"] > 0 + assert record["scratch_filesystem_used_bytes"] > 0 + + +@pytest.mark.parametrize("family", ["sql", "path", "command"]) +def test_task_boundary_real_local_oracles(tmp_path, family): + sides, demos = probe.materialise(tmp_path / family, family) + result = probe._verify_task_pair(sides, demos["real"], family) + assert result.outcome == "demonstrated", result + assert all(len(side.runs) == 3 for side in result.sides) + + +def test_task_boundary_xss_uses_browser_mode(tmp_path, monkeypatch): + from openultrasast.search.oracles import BrowserExecutor + + calls = [] + + def capture(self, document, nonce): + assert self.task_boundary and self.no_sandbox + calls.append(document) + return '' in document + + monkeypatch.setattr(BrowserExecutor, "__call__", capture) + sides, demos = probe.materialise(tmp_path / "xss", "xss") + result = probe._verify_task_pair(sides, demos["real"], "xss") + assert result.outcome == "demonstrated" + assert result.chromium_no_sandbox + assert all(side.chromium_no_sandbox for side in result.sides) + assert len(calls) == 6 + + +@pytest.mark.parametrize("mismatch", ["real", "observer_access", "revision_patch", "exception"]) +def test_task_boundary_mismatch_names_first_and_finishes(tmp_path, monkeypatch, mismatch): + from openultrasast.search.verify import VerificationRecord + + # On AX the diagnostics select the boundary before the failed bwrap preflight. + facts = {name: {"exit_code": 1} for name in ("user", "pid", "net", "ipc", "uts", "mount")} + monkeypatch.setattr(probe, "namespace_facts", lambda: facts) + monkeypatch.setattr(probe, "image_facts", lambda: {}) + monkeypatch.setattr(probe._sandbox, "isolation_check", lambda **kw: pytest.fail("must skip preflight")) + + def result(sides, demo, family): + if mismatch == "exception": + raise OSError("executor unavailable") + demonstrated = demo.name == "real" + if demo.name == mismatch: + demonstrated = not demonstrated + return VerificationRecord("demonstrated" if demonstrated else "inconclusive", (), 1, "", "task-boundary") + + monkeypatch.setattr(probe, "_verify_task_pair", result) + record = probe.probe(tmp_path) + assert record["status"] == "instrument_failure" + assert record["exception"].startswith(f"sql/{'real' if mismatch == 'exception' else mismatch}:") + assert record["namespaces"] == facts + assert record["isolation_check"] == {"status": "skipped", "reason": "task-boundary"} + assert sum(len(demos) for demos in record["oracles"].values()) == 15 diff --git a/tests/test_search_task_storage.py b/tests/test_search_task_storage.py index 3b07194..98042de 100644 --- a/tests/test_search_task_storage.py +++ b/tests/test_search_task_storage.py @@ -142,7 +142,7 @@ def test_preparation_builds_before_export_and_exports_products(workspace, monkey blobs, commands = [], [] monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data)) - def build(argv, timeout, limits): + def build(argv, timeout, limits, **kwargs): commands.append(argv) products = Path(argv[argv.index("--target") + 1]) assert products.is_relative_to(workspace) @@ -286,14 +286,19 @@ def test_preparation_with_many_host_processes(workspace, monkeypatch): @pytest.mark.parametrize("memory_bytes", [None, 256 * 1024**2, 5 * 1024**3]) def test_prepare_build_virtual_memory(workspace, monkeypatch, memory_bytes): - from openultrasast.search.demo import BUILD_RECIPES repo = workspace / "checkout" repo.mkdir() source = "import mmap; region=mmap.mmap(-1,512*1024**2); print(len(region))" (repo / "app.py").write_text(source) print("input_bytes=", len((repo / "app.py").read_bytes())) - monkeypatch.setitem(BUILD_RECIPES, "npm", (sys.executable, "-c", source)) + (repo / "package.json").write_text("{}") + fake_bin = workspace / "bin" + fake_bin.mkdir() + tool = fake_bin / "npm" + tool.write_text("#!" + sys.executable + "\n" + source) + tool.chmod(0o755) + monkeypatch.setitem(executor.SAFE_ENV, "PATH", str(fake_bin) + ":/usr/bin:/bin") app = executor.InProcessExecutor(repo, task_boundary=True) app.prepared_put_url = "https://files.example/output" blobs = [] @@ -317,3 +322,101 @@ def test_verifier_app_virtual_memory(workspace): demo.write_text(json.dumps(spec()["demo"])) result = verify_side_task(Side(repo), demo, "path") assert result.outcome == "observed", result + + +@pytest.mark.parametrize( + "recipe,manifest,extra,expected", + [ + ("pip", "pyproject.toml", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "PROJECT"]), + ("pip", "requirements.txt", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "MANIFEST"]), + ( + "pip", + "pyproject.toml", + {"requirements.txt": ""}, + ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "REQUIREMENTS"], + ), + ("npm", "package.json", {}, ["install", "--ignore-scripts"]), + ("npm", "package.json", {"package-lock.json": "{}"}, ["ci", "--ignore-scripts"]), + ("composer", "composer.json", {}, ["install", "--no-interaction", "--no-plugins", "--no-scripts"]), + ("maven", "pom.xml", {}, ["-q", "-DskipTests", "package", "-f", "MANIFEST"]), + ], +) +@pytest.mark.parametrize("directory", [False, True]) +def test_recipe_command_with_real_fake_tool(workspace, monkeypatch, recipe, manifest, extra, expected, directory): + repo = workspace / "checkout" + project = repo / "project" + project.mkdir(parents=True) + (repo / "app.py").write_text("print('ready')") + contents = '[build-system]\nbuild-backend="setuptools.build_meta"\n' if manifest == "pyproject.toml" and not extra else "" + (project / manifest).write_text(contents) + for name, content in extra.items(): + (project / name).write_text(content) + tools = workspace / "bin" + tools.mkdir() + tool = tools / {"pip": "python3", "maven": "mvn"}.get(recipe, recipe) + tool.write_text( + "#!" + sys.executable + "\nimport json, os, sys\nfrom pathlib import Path\n" + "Path('invocation.json').write_text(json.dumps([sys.argv[1:], os.getcwd()]))\n" + ) + tool.chmod(0o755) + monkeypatch.setitem(executor.SAFE_ENV, "PATH", str(tools) + ":/usr/bin:/bin") + monkeypatch.setattr(executor.URLTransport, "put", lambda *args: None) + app = executor.InProcessExecutor(repo, task_boundary=True) + app.prepared_put_url = "https://files.example/output" + request = spec() + request["demo"]["build"] = {"recipe": recipe, "arguments": ["project" if directory else "project/" + manifest]} + result = app.submit("prepare_verification", request) + assert result["status"] == "ok", result + argv, cwd = json.loads((project / "invocation.json").read_text()) + assert cwd == str(project) + replacements = {"PROJECT": str(project), "MANIFEST": str(project / manifest), "REQUIREMENTS": str(project / "requirements.txt")} + if "--target" in argv: + packages = argv[argv.index("--target") + 1] + assert Path(packages).name == "packages" and Path(packages).is_relative_to(workspace) + replacements["PACKAGES"] = packages + assert argv == [replacements.get(arg, arg) for arg in expected] + + +def test_failed_build_preserves_sanitized_tail(workspace, monkeypatch): + repo = workspace / "checkout" + repo.mkdir() + (repo / "gradlew").write_text( + "for i in $(seq 1 30); do echo line-$i >&2; done\necho 'https://example.test/?token=secret token=hidden' >&2\nexit 7\n" + ) + app = executor.InProcessExecutor(repo, task_boundary=True) + app.prepared_put_url = "https://files.example/output" + request = spec() + request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]} + result = app.submit("prepare_verification", request) + assert result["status"] == "could_not_build" and result["exit_code"] == 7 + assert result["phase"] == "build" + assert "line-30" in result["stderr"] and "line-1\n" not in result["stderr"] + assert "hidden" not in result["stderr"] and "https://" not in result["stderr"] + assert len(result["stderr"]) <= 1500 and len(result["stderr"].splitlines()) <= 20 + + +@pytest.mark.parametrize("mode", ["cli", "http"]) +def test_verifier_start_failure_has_diagnostics(workspace, mode): + repo = workspace / "checkout" + repo.mkdir() + (repo / "app.py").write_text( + "import sys\nprint('start failed token=hidden https://example.test/?secret=x', file=sys.stderr)\nsys.exit(9)" + ) + demo = workspace / "demo.json" + value = spec()["demo"] + value["start"]["mode"] = mode + if mode == "http": + value["start"]["port"] = 19001 + value["steps"] = [{"type": "http", "method": "GET", "path": "/"}] + demo.write_text(json.dumps(value)) + result = verify_side_task(Side(repo), demo, "path") + assert result.outcome == "could_not_run", result + assert result.phase == "start" and result.exit_code == 9 + assert "start failed" in result.stderr and "hidden" not in result.stderr and "https://" not in result.stderr + + +@pytest.mark.parametrize("secret", ['token="hidden"', '"password": "hidden"', "Bearer hidden", "ghp_hidden", "npm_hidden"]) +def test_command_failure_sanitizes_tokens_and_caps_tail(secret): + failure = task_storage.CommandFailure("build", 1, "old\n" * 50 + "x" * 2000 + "\n" + secret) + assert "hidden" not in failure.stderr and "old" not in failure.stderr + assert len(failure.stderr) <= 1500