marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 19:59:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

plane(2.1): Delivery -- presigned store URLs, collection, state mirrored to the store

Browse Source 
src/openultrasast/plane/delivery.py: Delivery(store, run, base) mints a presigned
PUT for runs/<run>/<task>/output.tar and a GET per declared input
(runs/<run>/<producer>/<artifact>), expiry OUSAST_TASK_TIMEOUT + 600 s;
delivered() asks the store for the exact key; collect() downloads, extracts
under the run directory with the receiver's member checks (no symlink, hard
link or escape; nothing written on a rejection), re-puts the declared outputs
as single objects and writes done.json; put_input() for inputs no task
produces; save_state()/load_state() mirror state.json. The reconciler opens the
store first (no fallback; the CLI reports a store that cannot open and submits
nothing), mirrors state.json after every change and seeds a rerun with no local
state from the store (the 2.1 bullet's state mirroring is what gives the new
module its importer). The reconciler crossed 500 lines here (517) rather than at
2.4 as 1.6 projected: the bound moves to 550 with the reason in the assertion,
as 1.6 decided. tests/plane_fake_store.py is the S3-shaped fake (objects in a
dict, presigned URLs on a local server); tests/test_plane_delivery.py covers the
body, expiry, polling, rejection, declared-only re-put, state seeding and that
no URL reaches a log. Module-audit manifest regenerated (148/5, no orphan).

Gates: pytest 2020 passed, 42 skipped, exit 0; ruff check 0; ruff format 0;
mypy 0.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-02 21:05:48 +02:00
co-authored by Claude Fable 5.1
parent 79296bb4a4
commit 35d8032c0a
8 changed files with 516 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openultrasast/cli.py
+7 -1
View File
@@ -1394,8 +1394,14 @@ def _plane(args: argparse.Namespace) -> int:
print(f"plane: {exc}", file=sys.stderr)
return 2
if args.plane_command == "run":
from .plane.memory import MemoryStoreError
_plane_memory("seed", args.run_manifest)
result = reconciler.run(args.run_manifest, workers=args.workers, ax=args.ax, profile=profile)
try:
result = reconciler.run(args.run_manifest, workers=args.workers, ax=args.ax, profile=profile)
except MemoryStoreError as exc: # the store is the state of record: no fallback, the Run is not submitted
print(f"memory store {profile.memory}: {exc}", file=sys.stderr)
return 1
print(f"run finished: {result}")
_plane_memory("ingest", args.run_manifest)
return 0 if result == "done" else 1
src/openultrasast/plane/delivery.py
+173
View File
@@ -0,0 +1,173 @@
"""Task artifacts through the memory store (plane-on-kubernetes design section 3, Req 3.1).
A task's runner PUTs its output tar to a presigned URL the reconciler minted, and GETs each declared input from
another; the URLs travel in the start request's ``delivery`` field (a presigned URL is a bearer capability, so it
goes where the credentials go, never into an ActorTemplate's env). The reconciler polls the store for the tar,
extracts it under the run directory with the member checks the old receiver had, re-puts the task's declared
``outputs[]`` as single objects so a consumer's URL names exactly one artifact, and writes ``done.json``. The run's
``state.json`` is mirrored to the store after each change and seeded from it on a rerun elsewhere.
Store layout under the 30-day ``runs/`` expiry rule (``memory.RUNS_RULE_ID``)::
runs/<run>/state.json the reconciler's state, mirrored
runs/<run>/<task>/output.tar the runner's PUT
runs/<run>/<task>/<artifact> each declared output, re-put by the reconciler
runs/<run>/<task>/done.json status, bytes, file count, finished
runs/<run>/inputs/<name> an input no task produces (``put_input``)
Nothing here logs or prints a URL.
"""
from __future__ import annotations
import io
import json
import os
import re
import tarfile
from collections.abc import Iterable, Mapping
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from .memory import RUNS_PREFIX, MemoryStore
__all__ = ["DONE", "GRACE_SECONDS", "OUTPUT_TAR", "STATE", "Delivery", "DeliveryError", "expiry", "input_var"]
OUTPUT_TAR = "output.tar"
DONE = "done.json"
STATE = "state.json"
INPUTS_DIR = "inputs"
GRACE_SECONDS = 600 # a URL outlives the task timeout by this much; a re-start after a re-resume re-mints anyway
_NAME = re.compile(r"[^A-Z0-9]+")
class DeliveryError(RuntimeError):
"""A delivered tar that cannot be accepted, or an object that is not there; the message never carries a URL."""
def expiry(environ: Mapping[str, str] | None = None) -> timedelta:
"""``OUSAST_TASK_TIMEOUT`` (7200 s) plus :data:`GRACE_SECONDS`."""
env = os.environ if environ is None else environ
return timedelta(seconds=float(env.get("OUSAST_TASK_TIMEOUT") or 7200) + GRACE_SECONDS)
def input_var(name: str) -> str:
"""The ``OUSAST_INPUT_<NAME>`` suffix of a Run input name, as the reconciler renders it."""
return _NAME.sub("_", name.upper())
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
class Delivery:
"""Presigned URLs, polling, collection and state mirroring for one Run on one store."""
def __init__(self, store: MemoryStore, run: str, base: Path, *, expires: timedelta | None = None) -> None:
self.store, self.run, self.base = store, run, base
self.expires = expires or expiry()
def key(self, *parts: str) -> str:
return f"{RUNS_PREFIX}{self.run}/" + "/".join(parts)
@property
def host(self) -> str | None:
"""The hostname the runner will dial (for the task's egress policy); None for a store without one."""
return urlsplit(self.store.presign_get(self.key(STATE), timedelta(seconds=60))).hostname
# --- the start body -------------------------------------------------------------------------------------------
def put_url(self, task: str) -> str:
return self.store.presign_put(self.key(task, OUTPUT_TAR), self.expires)
def input_urls(self, inputs: Mapping[str, str]) -> dict[str, str]:
"""``{OUSAST_INPUT_<NAME> suffix: presigned GET of runs/<run>/<producer>/<artifact>}``."""
out: dict[str, str] = {}
for name, ref in inputs.items():
producer, _, artifact = ref.partition("/")
if not producer or not artifact:
raise DeliveryError(f"input {name}: {ref!r} is not <producer>/<artifact>")
out[input_var(name)] = self.store.presign_get(self.key(producer, artifact), self.expires)
return out
def body(self, task: str, inputs: Mapping[str, str]) -> dict[str, Any]:
"""The start request's ``delivery`` field for ``task``."""
return {"put": self.put_url(task), "inputs": self.input_urls(inputs)}
# --- completion ----------------------------------------------------------------------------------------------
def delivered(self, task: str) -> bool:
"""Whether the runner's tar is in the store (an exact-key listing, the store's HEAD)."""
key = self.key(task, OUTPUT_TAR)
return key in self.store._keys(key)
def collect(self, task: str, outputs: Iterable[str]) -> tuple[dict[str, Any] | None, int]:
"""Download and extract the tar under ``<base>/<task>/``, re-put the declared ``outputs``, write
``done.json``; returns (the extracted ``summary.json`` or None, the tar's byte size)."""
got = self.store._get(self.key(task, OUTPUT_TAR))
if got is None:
raise DeliveryError(f"task {task}: output.tar is not in the store")
data = got[0]
dest = (self.base / task).resolve()
try:
with tarfile.open(fileobj=io.BytesIO(data), mode="r:") as tar:
members = tar.getmembers()
for m in members:
if m.issym() or m.islnk() or not (dest / m.name).resolve().is_relative_to(dest):
raise DeliveryError(f"task {task}: rejected member {m.name}")
dest.mkdir(parents=True, exist_ok=True)
tar.extractall(dest, members) # every member checked above
except tarfile.TarError as exc:
raise DeliveryError(f"task {task}: bad tar: {exc}") from None
published = self.publish(task, outputs)
summary = _read_json(dest / "summary.json")
done = {
"task": task,
"status": summary.get("status") if isinstance(summary, dict) else None,
"bytes": len(data),
"files": sum(1 for m in members if m.isfile()),
"published": published,
"finished": _now(),
}
self.store._put(self.key(task, DONE), json.dumps(done, sort_keys=True).encode("utf-8"))
return (summary if isinstance(summary, dict) else None), len(data)
def publish(self, task: str, outputs: Iterable[str]) -> list[str]:
"""Re-put the task's declared outputs that exist under ``<base>/<task>/`` as single objects; the names put."""
put: list[str] = []
root = (self.base / task).resolve()
for artifact in outputs:
path = (root / artifact).resolve()
if path.is_relative_to(root) and path.is_file():
self.store._put(self.key(task, artifact), path.read_bytes())
put.append(artifact)
return put
def put_input(self, name: str, path: Path) -> str:
"""Store a file no task produces as ``runs/<run>/inputs/<name>``; returns its presigned GET URL."""
self.store._put(self.key(INPUTS_DIR, name), Path(path).read_bytes())
return self.store.presign_get(self.key(INPUTS_DIR, name), self.expires)
# --- state ---------------------------------------------------------------------------------------------------
def save_state(self, data: Mapping[str, Any]) -> None:
self.store._put(self.key(STATE), (json.dumps(data, indent=2) + "\n").encode("utf-8"))
def load_state(self) -> dict[str, Any] | None:
got = self.store._get(self.key(STATE))
if got is None:
return None
try:
loaded = json.loads(got[0].decode("utf-8"))
except ValueError:
return None
return loaded if isinstance(loaded, dict) else None
def _read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
src/openultrasast/plane/reconciler.py
+32 -4
View File
@@ -18,7 +18,10 @@ task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace'
inputs only). The receiver takes the runner's tar (``X-Ousast-Run``, ``X-Ousast-Task``) on
``OUSAST_ARTIFACT_PORT`` (18090), reached by its Service name through the egress gateway; between ``ax apply`` and
resume the task's egress policy is written (``egress.py``).
State: ``~/ousast-results/plane/<run>/`` (``OUSAST_RESULTS``), polled every ``OUSAST_POLL_SECONDS`` (3).
State: ``~/ousast-results/plane/<run>/`` (``OUSAST_RESULTS``), polled every ``OUSAST_POLL_SECONDS`` (3); ``state.json``
is mirrored to ``runs/<run>/state.json`` on the memory store after every change and a rerun with no local state
seeds it from there (``delivery.py``; plane-on-kubernetes design section 2). The store is opened first and is
required: there is no fallback.
"""
from __future__ import annotations
@@ -42,9 +45,11 @@ from urllib.parse import urlsplit
import yaml
from .delivery import Delivery
from .doctor import doctor
from .egress import RECEIVER_PORT, Egress, Receiver, policy_for
from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests
from .memory import open_store
from .profile import PlaneProfile, load_profile
from .router import Router, credentials, open_router
@@ -226,12 +231,25 @@ class _State:
path: Path
data: dict[str, Any]
lock: threading.Lock = field(default_factory=threading.Lock)
mirror: Callable[[Mapping[str, Any]], None] | None = None # the store copy; a failure is reported, never fatal
def set(self, task: str, **fields: object) -> None:
with self.lock:
self.data["tasks"].setdefault(task, {"status": "pending", "started": None, "finished": None, "usd": None, "model": None})
self.data["tasks"][task].update(fields)
_write_json(self.path, self.data)
self._flush()
def flush(self) -> None:
with self.lock:
self._flush()
def _flush(self) -> None:
_write_json(self.path, self.data)
if self.mirror is not None:
try:
self.mirror(self.data)
except Exception as exc: # noqa: BLE001 - the local file is written; the mirror is reported, the run goes on
print(f"state mirror to the store failed: {exc}", file=sys.stderr)
def status(self, task: str) -> str:
return str(self.data["tasks"].get(task, {}).get("status", "pending"))
@@ -357,8 +375,18 @@ def run(
run_spec, manifests = load_run(Path(run_manifest))
base = run_dir(run_spec.metadata.name, results_root)
base.mkdir(parents=True, exist_ok=True)
store = open_store(profile.memory) # first, before the lock and before ax sees anything: no fallback
delivery = Delivery(store, run_spec.metadata.name, base)
if not (base / "state.json").exists():
seeded = delivery.load_state() # a rerun from another machine or an empty results root
if seeded is not None:
_write_json(base / "state.json", seeded)
lock = _acquire_lock(base)
state = _State(base / "state.json", _read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}})
state = _State(
base / "state.json",
_read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}},
mirror=delivery.save_state,
)
port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT)
receiver = Receiver(run_spec.metadata.name, base, int(port), profile.kube_context) # the Service targets RECEIVER_PORT
threading.Thread(target=receiver.serve_forever, daemon=True).start()
@@ -409,7 +437,7 @@ def run(
statuses = [state.status(t.name) for t in run_spec.tasks]
result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished"
state.data.update(status=result, finished=_now())
_write_json(state.path, state.data)
state.flush()
return result