mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
Marius CiepluchandClaude Fable 5.1
0d3e6df31c
Archives are gzipped and uploaded in parts of at most 90 MB with per-part digests (edge request limit); pip installs ignore the task venv so transitive dependencies land in products; the verifier sets NODE_PATH and the Java classpath from products when starting the app. Co-Authored-By: Claude Fable 5.1 <[email protected]>
318 lines
13 KiB
Python
318 lines
13 KiB
Python
"""Key-free AX task entry for one declarative verification repetition."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import ipaddress
|
|
import json
|
|
import math
|
|
import os
|
|
import stat
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import asdict
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any, BinaryIO, cast
|
|
from urllib.parse import urlsplit
|
|
|
|
from . import task_storage
|
|
from .demo import validate_demo, validate_oracle
|
|
from .executor import clean_environment
|
|
from .verify import Side, SideRecord
|
|
|
|
|
|
def public_endpoint(url: str) -> str:
|
|
parts = urlsplit(url)
|
|
host = parts.hostname or ""
|
|
try:
|
|
ipaddress.ip_address(host)
|
|
is_ip = True
|
|
except ValueError:
|
|
is_ip = False
|
|
if (
|
|
parts.scheme != "https"
|
|
or not host
|
|
or is_ip
|
|
or "." not in host
|
|
or host.endswith((".localhost", ".local", ".internal"))
|
|
or parts.port not in (None, 443)
|
|
or parts.username
|
|
or parts.password
|
|
):
|
|
raise ValueError("task URL requires public HTTPS hostname")
|
|
return host
|
|
|
|
|
|
MAX_ARCHIVE_BYTES = 1024**3
|
|
MAX_PART_BYTES = 90_000_000
|
|
MAX_PARTS = (MAX_ARCHIVE_BYTES + MAX_PART_BYTES - 1) // MAX_PART_BYTES
|
|
MAX_MANIFEST_BYTES = 65536
|
|
MAX_CHECKOUT_BYTES = 2 * 1024**3
|
|
MAX_FILES = 20000
|
|
MAX_SPEC_BYTES = 70000
|
|
MAX_RESULT_BYTES = 65536
|
|
|
|
|
|
CACHE_NAMES = {".npm", ".pip", ".m2", ".composer", ".gradle", "__pycache__", ".git"}
|
|
|
|
|
|
def bundle_ignore(directory: str, names: list[str]) -> set[str]:
|
|
return {name for name in names if name in CACHE_NAMES or (Path(directory).name == "node_modules" and name == ".cache")}
|
|
|
|
|
|
class ArchiveLimit(ValueError):
|
|
def __init__(self, size: int, directories: dict[str, int]) -> None:
|
|
self.archive_bytes = size
|
|
largest = sorted(directories.items(), key=lambda item: (-item[1], item[0]))[:5]
|
|
super().__init__(
|
|
f"archive exceeds size limit: {size} bytes > {MAX_ARCHIVE_BYTES} bytes; top 5 directories: "
|
|
+ ", ".join(f"{name} ({count} bytes)" for name, count in largest)
|
|
)
|
|
|
|
|
|
def pack_checkout(root: Path, output: BinaryIO | None = None) -> bytes:
|
|
root = Path(root)
|
|
if root.is_symlink() or not root.is_dir():
|
|
raise ValueError("checkout must be a real directory")
|
|
buffer = output if output is not None else io.BytesIO()
|
|
total, count = 0, 0
|
|
directories: dict[str, int] = {}
|
|
with tarfile.open(fileobj=buffer, mode="w:gz", compresslevel=6) as archive:
|
|
for parent, dirs, files in os.walk(root, followlinks=False):
|
|
ignored = bundle_ignore(parent, dirs + files)
|
|
dirs[:] = sorted(set(dirs) - ignored)
|
|
for name in sorted(dirs + [name for name in files if name not in ignored]):
|
|
path = Path(parent) / name
|
|
mode = path.lstat().st_mode
|
|
count += 1
|
|
if count > MAX_FILES or not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
|
|
raise ValueError("checkout contains too many files or special files")
|
|
size = path.stat().st_size if path.is_file() else 0
|
|
total += size
|
|
if total > MAX_CHECKOUT_BYTES:
|
|
raise ValueError("checkout exceeds size limit")
|
|
for directory in path.relative_to(root).parents:
|
|
if str(directory) != ".":
|
|
directories[str(directory)] = directories.get(str(directory), 0) + size
|
|
archive.add(path, arcname=str(path.relative_to(root)), recursive=False)
|
|
if buffer.tell() > MAX_ARCHIVE_BYTES:
|
|
raise ArchiveLimit(buffer.tell(), directories)
|
|
return cast(io.BytesIO, buffer).getvalue() if output is None else b""
|
|
|
|
|
|
def upload_parts(source: BinaryIO, put: Any, urls: list[str]) -> bytes:
|
|
"""Publish bounded gzip fragments, then return the ordered digest manifest."""
|
|
source.seek(0, 2)
|
|
size = source.tell()
|
|
if not 0 < size <= MAX_ARCHIVE_BYTES:
|
|
raise ValueError("archive exceeds size limit")
|
|
if len(urls) < (size + MAX_PART_BYTES - 1) // MAX_PART_BYTES:
|
|
raise ValueError("missing archive part URLs")
|
|
source.seek(0)
|
|
parts: list[dict[str, Any]] = []
|
|
while block := source.read(MAX_PART_BYTES):
|
|
index = len(parts)
|
|
put(urls[index], block)
|
|
parts.append({"index": index, "size": len(block), "sha256": hashlib.sha256(block).hexdigest()})
|
|
return json.dumps({"format": "gzip-parts-v1", "size": size, "parts": parts}).encode()
|
|
|
|
|
|
def assemble_parts(data: bytes, output: BinaryIO, get: Any) -> None:
|
|
"""Validate the complete manifest before fetching, and each part before use."""
|
|
if len(data) > MAX_MANIFEST_BYTES:
|
|
raise ValueError("oversized archive manifest")
|
|
manifest = json.loads(data)
|
|
if not isinstance(manifest, dict) or manifest.get("format") != "gzip-parts-v1":
|
|
raise ValueError("invalid archive manifest")
|
|
parts = manifest.get("parts")
|
|
size = manifest.get("size")
|
|
if type(size) is not int or not 0 < size <= MAX_ARCHIVE_BYTES:
|
|
raise ValueError("archive exceeds size limit")
|
|
if not isinstance(parts, list) or not 1 <= len(parts) <= MAX_PARTS:
|
|
raise ValueError("invalid archive parts")
|
|
total = 0
|
|
for index, part in enumerate(parts):
|
|
if (
|
|
not isinstance(part, dict)
|
|
or type(part.get("index")) is not int
|
|
or part["index"] != index
|
|
or type(part.get("size")) is not int
|
|
or not 0 < part["size"] <= MAX_PART_BYTES
|
|
or not isinstance(part.get("sha256"), str)
|
|
or len(part["sha256"]) != 64
|
|
):
|
|
raise ValueError("invalid archive part")
|
|
total += part["size"]
|
|
if total != size:
|
|
raise ValueError("archive part sizes mismatch")
|
|
for part in parts:
|
|
block = get(part)
|
|
if len(block) != part["size"] or hashlib.sha256(block).hexdigest() != part["sha256"]:
|
|
raise ValueError("archive part digest mismatch")
|
|
output.write(block)
|
|
output.seek(0)
|
|
|
|
|
|
def extract_checkout(data: bytes | BinaryIO, destination: Path) -> None:
|
|
compressed_source = io.BytesIO(data) if isinstance(data, bytes) else data
|
|
compressed_source.seek(0, 2)
|
|
size = compressed_source.tell()
|
|
compressed_source.seek(0)
|
|
if size > MAX_ARCHIVE_BYTES:
|
|
raise ValueError("archive exceeds size limit")
|
|
# Bound expansion before tarfile parses PAX headers, using disk rather than
|
|
# allocating the entire expanded tree in worker RAM.
|
|
with tempfile.TemporaryFile() as expanded:
|
|
with gzip.GzipFile(fileobj=compressed_source) as compressed:
|
|
maximum = MAX_CHECKOUT_BYTES + MAX_FILES * 4096
|
|
while block := compressed.read(min(65536, maximum - expanded.tell() + 1)):
|
|
expanded.write(block)
|
|
if expanded.tell() > maximum:
|
|
raise ValueError("expanded checkout exceeds size limit")
|
|
expanded.seek(0)
|
|
destination.mkdir(parents=True)
|
|
total, seen = 0, set()
|
|
with tarfile.open(fileobj=expanded, mode="r:") as archive:
|
|
for index, member in enumerate(archive):
|
|
path = PurePosixPath(member.name)
|
|
if (
|
|
index >= MAX_FILES
|
|
or path.is_absolute()
|
|
or ".." in path.parts
|
|
or not path.parts
|
|
or "\\" in member.name
|
|
or member.name in seen
|
|
or not (member.isfile() or member.isdir())
|
|
):
|
|
raise ValueError("unsafe checkout archive member")
|
|
seen.add(member.name)
|
|
total += member.size
|
|
if member.size < 0 or total > MAX_CHECKOUT_BYTES:
|
|
raise ValueError("expanded checkout exceeds size limit")
|
|
target = destination.joinpath(*path.parts)
|
|
if member.isdir():
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
else:
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
source = archive.extractfile(member)
|
|
if source is None:
|
|
raise ValueError("missing regular checkout member")
|
|
with source, target.open("xb") as output:
|
|
remaining = member.size
|
|
while remaining:
|
|
block = source.read(min(65536, remaining))
|
|
if not block:
|
|
raise ValueError("truncated checkout member")
|
|
output.write(block)
|
|
remaining -= len(block)
|
|
target.chmod(0o755 if member.mode & 0o111 else 0o644)
|
|
if not seen:
|
|
raise ValueError("empty checkout")
|
|
print(f"checkout_bytes={total} checkout_entries={len(seen)}", flush=True)
|
|
|
|
|
|
def validate_spec(spec: Any) -> dict[str, Any]:
|
|
if not isinstance(spec, dict) or set(spec) != {"demo", "family", "timeout_seconds"}:
|
|
raise ValueError("invalid verification spec")
|
|
if len(json.dumps(spec, allow_nan=False).encode()) > MAX_SPEC_BYTES:
|
|
raise ValueError("oversized verification spec")
|
|
validate_demo(spec["demo"])
|
|
if spec["demo"]["build"]["recipe"] != "none":
|
|
raise ValueError("verifier requires executor-built archive; builds are forbidden")
|
|
validate_oracle(spec["family"], spec["demo"]["oracle"])
|
|
timeout = spec["timeout_seconds"]
|
|
if type(timeout) is not int or not math.isfinite(timeout) or not 1 <= timeout <= 300:
|
|
raise ValueError("invalid verification timeout")
|
|
return cast(dict[str, Any], spec)
|
|
|
|
|
|
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, *args: Any, **kwargs: Any) -> None:
|
|
raise ValueError("presigned redirects refused")
|
|
|
|
|
|
def download(url: str, limit: int, *, readiness_budget: float = 60) -> bytes:
|
|
"""The first object read tolerates the task egress policy's startup delay."""
|
|
public_endpoint(url)
|
|
end = time.monotonic() + readiness_budget
|
|
delay = 1.0
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect)
|
|
while True:
|
|
try:
|
|
with opener.open(url, timeout=max(0.01, min(30, end - time.monotonic()))) as response:
|
|
data = response.read(limit + 1)
|
|
if len(data) > limit:
|
|
raise ValueError("oversized task input")
|
|
return cast(bytes, data)
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code != 403 and not 500 <= exc.code < 600:
|
|
raise OSError("task input download failed") from None
|
|
except (urllib.error.URLError, OSError):
|
|
pass
|
|
if time.monotonic() >= end:
|
|
raise OSError("task input egress not ready before download deadline") from None
|
|
time.sleep(min(delay, end - time.monotonic()))
|
|
delay = min(delay * 2, 8)
|
|
|
|
|
|
def upload(url: str, data: bytes) -> None:
|
|
public_endpoint(url)
|
|
if len(data) > MAX_RESULT_BYTES:
|
|
raise ValueError("oversized task result")
|
|
request = urllib.request.Request(url, data=data, method="PUT")
|
|
with urllib.request.build_opener(_NoRedirect).open(request, timeout=30) as response:
|
|
if not 200 <= response.status < 300:
|
|
raise ValueError("result upload failed")
|
|
|
|
|
|
def run_side(side: Side, demo: Path, family: str, timeout_seconds: int) -> SideRecord:
|
|
from openultrasast.search.verify import verify_side_task
|
|
|
|
return verify_side_task(side, demo, family, timeout_seconds)
|
|
|
|
|
|
def task_main() -> None:
|
|
if any(
|
|
value and any(word in key.upper() for word in ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL"))
|
|
for key, value in os.environ.items()
|
|
):
|
|
raise ValueError("verification task refuses inherited credentials")
|
|
urls = {key: os.environ[key] for key in ("VERIFY_INPUT_URL", "RESULT_URL")}
|
|
for url in urls.values():
|
|
public_endpoint(url)
|
|
scratch_limit = task_storage.configure()
|
|
os.environ.clear()
|
|
os.environ.update(clean_environment())
|
|
os.environ["OUSAST_SCRATCH_BYTES"] = str(scratch_limit)
|
|
archive = download(urls["VERIFY_INPUT_URL"], MAX_ARCHIVE_BYTES)
|
|
with tempfile.TemporaryDirectory(prefix="ousast-verify-task-") as temporary:
|
|
root = Path(temporary)
|
|
try:
|
|
if archive.startswith(b"\x1f\x8b"):
|
|
extract_checkout(archive, root / "input")
|
|
else:
|
|
with tempfile.TemporaryFile() as compressed:
|
|
assemble_parts(archive, compressed, lambda part: download(part["url"], part["size"]))
|
|
extract_checkout(compressed, root / "input")
|
|
task_storage.check(task_storage.WORKSPACE if task_storage.WORKSPACE.is_dir() else root)
|
|
bundle = root / "input"
|
|
if {p.name for p in bundle.iterdir()} != {"checkout", "products", "spec.json"}:
|
|
raise ValueError("expected built checkout, products and spec in one executor archive")
|
|
spec = validate_spec(json.loads((bundle / "spec.json").read_bytes()))
|
|
demo = root / "demo"
|
|
demo.mkdir()
|
|
(demo / "demo.json").write_text(json.dumps(spec["demo"]))
|
|
result = run_side(Side(bundle / "checkout", products=bundle / "products"), demo, spec["family"], spec["timeout_seconds"])
|
|
except task_storage.ScratchLimit as exc:
|
|
result = SideRecord("could_not_build", (), 0, task_storage.exception_reason(exc), isolation_mode="task-boundary")
|
|
upload(urls["RESULT_URL"], json.dumps(asdict(result)).encode())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
task_main()
|