mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
merge(plane-on-kubernetes): groups 1-2 -- cluster profile, no host literals, delivery through the S3 store
Kind proofs: records 01 and 02. Task 2.6 (receiver removal) is paused while the safety net and the decision engine come first. deployment.md conflicts resolved in the plain-language style; corrects the RuntimeClass claim (Substrate runs runsc inside its worker pod). Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
+29
-10
@@ -323,16 +323,21 @@ def _main(argv: list[str] | None) -> int:
|
||||
learn_labels.add_argument("--include-title", action="store_true", help="the title-tier pairs as well (a sensitivity arm)")
|
||||
learn_labels.add_argument("--no-assumed-benign", action="store_true", help="skip mining the assumed-benign history source")
|
||||
add_learn_commands(learn_sub)
|
||||
plane = subparsers.add_parser("plane", help="maintainer: the ax-backed service plane on this host's kind cluster")
|
||||
plane = subparsers.add_parser("plane", help="maintainer: the ax-backed service plane on the profile's cluster (ops/k8s/profiles/)")
|
||||
profile_help = "the PlaneProfile: a name under ops/k8s/profiles/ or a .toml path (default OUSAST_PLANE_PROFILE, else kind)"
|
||||
plane_sub = plane.add_subparsers(dest="plane_command", required=True)
|
||||
plane_run = plane_sub.add_parser("run", help="execute a Run manifest on ax; a rerun skips done tasks")
|
||||
plane_run.add_argument("run_manifest", type=Path)
|
||||
plane_run.add_argument("--workers", type=int, default=1, help="tasks in flight at once (serialize labels never overlap)")
|
||||
plane_run.add_argument("--ax", default="ax", help="path of the ax CLI")
|
||||
plane_run.add_argument("--profile", help=profile_help)
|
||||
plane_status = plane_sub.add_parser("status", help="per-task status and the token attribution table of a run")
|
||||
plane_status.add_argument("run")
|
||||
plane_status.add_argument("--units", action="store_true", help="per-unit rows from each task's units.jsonl")
|
||||
plane_sub.add_parser("doctor", help="check kind, Agent Substrate, the ax controller and the runner image")
|
||||
plane_doctor = plane_sub.add_parser(
|
||||
"doctor", help="check the profile's cluster, Agent Substrate, the ax controller, the runner image and the memory store"
|
||||
)
|
||||
plane_doctor.add_argument("--profile", help=profile_help)
|
||||
plane_remember = plane_sub.add_parser("remember", help="ingest a run's memory rows into the store (OUSAST_MEMORY)")
|
||||
plane_remember.add_argument("run")
|
||||
plane_remember.add_argument("--plane", type=Path, default=Path("plane"), help="where runs/<run>.yaml names population and split")
|
||||
@@ -370,9 +375,11 @@ def _main(argv: list[str] | None) -> int:
|
||||
)
|
||||
plane_harvest.add_argument("--ceiling", type=float, default=10.0, help="USD: each Run's task budgets sum to at most this")
|
||||
plane_harvest.add_argument("--runner-image", type=Path, help="re-pin the task templates to the image in this file first (ops/ax/up.sh)")
|
||||
plane_harvest.add_argument("--profile", help=profile_help)
|
||||
plane_ws = plane_sub.add_parser("workspaces", help="one Workspace manifest per case pin of a population file")
|
||||
plane_ws.add_argument("population", type=Path)
|
||||
plane_ws.add_argument("--out", type=Path, default=Path("plane/workspaces"))
|
||||
plane_ws.add_argument("--profile", help=profile_help)
|
||||
plane_ws.add_argument("--validation-set", type=Path, help="JSON {case: [[path, function], ...]}: emit that set's increment instead")
|
||||
plane_ws.add_argument("--candidates", type=Path, help="with --validation-set: per-case candidate records (independent-v2-modelsinks)")
|
||||
plane_ws.add_argument("--triage", type=Path, help="with --validation-set: recorded batched-check scans (<case>--vulnerable_a.jsonl)")
|
||||
@@ -1529,10 +1536,22 @@ def _learn_labels(args: argparse.Namespace) -> int:
|
||||
|
||||
def _plane(args: argparse.Namespace) -> int:
|
||||
from .plane import reconciler # lazy: the plane is a maintainer surface, not the scan path
|
||||
from .plane.profile import ProfileError, load_profile
|
||||
|
||||
try:
|
||||
profile = load_profile(getattr(args, "profile", None))
|
||||
except ProfileError as exc:
|
||||
print(f"plane: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
if args.plane_command == "run":
|
||||
from .plane.memory import MemoryStoreError
|
||||
|
||||
_plane_memory("seed", args.run_manifest)
|
||||
result = reconciler.run(args.run_manifest, workers=args.workers, ax=args.ax)
|
||||
try:
|
||||
result = reconciler.run(args.run_manifest, workers=args.workers, ax=args.ax, profile=profile)
|
||||
except MemoryStoreError as exc: # the store is the state of record: no fallback, the Run is not submitted
|
||||
print(f"memory store {profile.memory}: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"run finished: {result}")
|
||||
_plane_memory("ingest", args.run_manifest)
|
||||
return 0 if result == "done" else 1
|
||||
@@ -1547,14 +1566,14 @@ def _plane(args: argparse.Namespace) -> int:
|
||||
if args.plane_command == "alerts-engine":
|
||||
return _plane_alerts_engine(args)
|
||||
if args.plane_command == "harvest":
|
||||
return _plane_harvest(args)
|
||||
return _plane_harvest(args, profile.atespace)
|
||||
if args.plane_command == "doctor":
|
||||
checks = reconciler.doctor()
|
||||
checks = reconciler.doctor(profile)
|
||||
for name, ok, text in checks:
|
||||
print(f"[{'ok' if ok else 'FAIL'}] {name}: {text}")
|
||||
return 0 if all(ok for _, ok, _ in checks) else 1
|
||||
if args.validation_set is not None:
|
||||
return _plane_increment(args)
|
||||
return _plane_increment(args, profile.atespace)
|
||||
from .plane.workspaces import workspaces
|
||||
|
||||
written = workspaces(args.population, args.out)
|
||||
@@ -1562,7 +1581,7 @@ def _plane(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _plane_harvest(args: argparse.Namespace) -> int:
|
||||
def _plane_harvest(args: argparse.Namespace, atespace: str) -> int:
|
||||
from .plane.generate import repin_templates
|
||||
from .plane.harvest import write_harvest
|
||||
|
||||
@@ -1571,7 +1590,7 @@ def _plane_harvest(args: argparse.Namespace) -> int:
|
||||
command = f"ousast plane harvest --labels {args.labels.name} --name {args.name} --ceiling {args.ceiling:g}"
|
||||
files = write_harvest(
|
||||
args.labels, args.plane, name=args.name, command=command, catalog=args.catalog, cache=args.cache, templates=args.templates,
|
||||
ceiling=args.ceiling,
|
||||
ceiling=args.ceiling, atespace=atespace,
|
||||
) # fmt: skip
|
||||
for relative in sorted(k for k in files if k.startswith("runs/")):
|
||||
print(f"{args.plane / relative}: {files[relative].splitlines()[1].lstrip('# ')}")
|
||||
@@ -1652,7 +1671,7 @@ def _plane_memory(
|
||||
return 0
|
||||
|
||||
|
||||
def _plane_increment(args: argparse.Namespace) -> int:
|
||||
def _plane_increment(args: argparse.Namespace, atespace: str) -> int:
|
||||
from .plane.generate import LOOP_MANIFEST, Loop, increment
|
||||
|
||||
if args.candidates is None or args.triage is None:
|
||||
@@ -1678,7 +1697,7 @@ def _plane_increment(args: argparse.Namespace) -> int:
|
||||
command += "".join(f" --qualify-population {p}" for p in loop.populations)
|
||||
written = increment(
|
||||
args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command,
|
||||
repos=args.repos, runner_image=args.runner_image, loop=loop,
|
||||
repos=args.repos, runner_image=args.runner_image, loop=loop, atespace=atespace,
|
||||
) # fmt: skip
|
||||
print(f"{len(written)} manifests written under {args.plane} (Run {args.run})")
|
||||
return 0
|
||||
|
||||
@@ -162,7 +162,9 @@ class EngineSession:
|
||||
return None
|
||||
payload = json.dumps({"query": code}).encode()
|
||||
call = urllib.request.Request(
|
||||
f"http://127.0.0.1:{self._port}/query-sync", data=payload, headers={"Content-Type": "application/json"}
|
||||
f"http://127.0.0.1:{self._port}/query-sync", # loopback: the Joern server this process started
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(call, timeout=timeout) as response: # noqa: S310 -- fixed loopback URL
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Task artifacts through the memory store (plane-on-kubernetes design section 3, Req 3.1).
|
||||
|
||||
A task's runner PUTs its output tar to a presigned URL the reconciler minted, and GETs each declared input from
|
||||
another; the URLs travel in the start request's ``delivery`` field (a presigned URL is a bearer capability, so it
|
||||
goes where the credentials go, never into an ActorTemplate's env). The reconciler polls the store for the tar,
|
||||
extracts it under the run directory with the member checks the old receiver had, re-puts the task's declared
|
||||
``outputs[]`` as single objects so a consumer's URL names exactly one artifact, and writes ``done.json``. The run's
|
||||
``state.json`` is mirrored to the store after each change and seeded from it on a rerun elsewhere.
|
||||
|
||||
Store layout under the 30-day ``runs/`` expiry rule (``memory.RUNS_RULE_ID``)::
|
||||
|
||||
runs/<run>/state.json the reconciler's state, mirrored
|
||||
runs/<run>/<task>/output.tar the runner's PUT
|
||||
runs/<run>/<task>/<artifact> each declared output, re-put by the reconciler
|
||||
runs/<run>/<task>/done.json status, bytes, file count, finished
|
||||
runs/<run>/inputs/<name> an input no task produces (``put_input``)
|
||||
|
||||
Nothing here logs or prints a URL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tarfile
|
||||
from collections.abc import Iterable, Mapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .manifests import Run
|
||||
from .memory import RUNS_PREFIX, MemoryStore
|
||||
|
||||
__all__ = ["DONE", "GRACE_SECONDS", "OUTPUT_TAR", "STATE", "Delivery", "DeliveryError", "declared", "expiry", "input_var"]
|
||||
|
||||
OUTPUT_TAR = "output.tar"
|
||||
DONE = "done.json"
|
||||
STATE = "state.json"
|
||||
INPUTS_DIR = "inputs"
|
||||
GRACE_SECONDS = 600 # a URL outlives the task timeout by this much; a re-start after a re-resume re-mints anyway
|
||||
_NAME = re.compile(r"[^A-Z0-9]+")
|
||||
|
||||
|
||||
class DeliveryError(RuntimeError):
|
||||
"""A delivered tar that cannot be accepted, or an object that is not there; the message never carries a URL."""
|
||||
|
||||
|
||||
def expiry(environ: Mapping[str, str] | None = None) -> timedelta:
|
||||
"""``OUSAST_TASK_TIMEOUT`` (7200 s) plus :data:`GRACE_SECONDS`."""
|
||||
env = os.environ if environ is None else environ
|
||||
return timedelta(seconds=float(env.get("OUSAST_TASK_TIMEOUT") or 7200) + GRACE_SECONDS)
|
||||
|
||||
|
||||
def input_var(name: str) -> str:
|
||||
"""The ``OUSAST_INPUT_<NAME>`` suffix of a Run input name, as the reconciler renders it."""
|
||||
return _NAME.sub("_", name.upper())
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def declared(run: Run) -> dict[str, list[str]]:
|
||||
"""Per Run task, the artifacts to publish as single objects: its ``outputs[]`` plus every ``<producer>/<artifact>``
|
||||
a consumer declares as an input on it (``summary.json`` of a producer is a usual one), in declaration order."""
|
||||
wanted: dict[str, list[str]] = {entry.name: list(entry.outputs) for entry in run.tasks}
|
||||
for entry in run.tasks:
|
||||
for ref in entry.inputs.values():
|
||||
producer, _, artifact = ref.partition("/")
|
||||
if producer in wanted and artifact and artifact not in wanted[producer]:
|
||||
wanted[producer].append(artifact)
|
||||
return wanted
|
||||
|
||||
|
||||
class Delivery:
|
||||
"""Presigned URLs, polling, collection and state mirroring for one Run on one store."""
|
||||
|
||||
def __init__(self, store: MemoryStore, run: str, base: Path, *, expires: timedelta | None = None) -> None:
|
||||
self.store, self.run, self.base = store, run, base
|
||||
self.expires = expires or expiry()
|
||||
|
||||
def key(self, *parts: str) -> str:
|
||||
return f"{RUNS_PREFIX}{self.run}/" + "/".join(parts)
|
||||
|
||||
@property
|
||||
def host(self) -> str | None:
|
||||
"""The hostname the runner will dial (for the task's egress policy); None for a store without one."""
|
||||
return urlsplit(self.store.presign_get(self.key(STATE), timedelta(seconds=60))).hostname
|
||||
|
||||
# --- the start body -------------------------------------------------------------------------------------------
|
||||
|
||||
def put_url(self, task: str) -> str:
|
||||
return self.store.presign_put(self.key(task, OUTPUT_TAR), self.expires)
|
||||
|
||||
def input_urls(self, inputs: Mapping[str, str]) -> dict[str, str]:
|
||||
"""``{OUSAST_INPUT_<NAME> suffix: presigned GET of runs/<run>/<producer>/<artifact>}``."""
|
||||
out: dict[str, str] = {}
|
||||
for name, ref in inputs.items():
|
||||
producer, _, artifact = ref.partition("/")
|
||||
if not producer or not artifact:
|
||||
raise DeliveryError(f"input {name}: {ref!r} is not <producer>/<artifact>")
|
||||
out[input_var(name)] = self.store.presign_get(self.key(producer, artifact), self.expires)
|
||||
return out
|
||||
|
||||
def body(self, task: str, inputs: Mapping[str, str]) -> dict[str, Any]:
|
||||
"""The start request's ``delivery`` field for ``task``."""
|
||||
return {"put": self.put_url(task), "inputs": self.input_urls(inputs)}
|
||||
|
||||
# --- completion ----------------------------------------------------------------------------------------------
|
||||
|
||||
def reset(self, task: str) -> None:
|
||||
"""Before an attempt: remove the previous attempt's tar and ``done.json`` so a rerun (an unfinished task
|
||||
resumed with a larger budget, an interrupted attempt) never completes on a stale object."""
|
||||
for name in (OUTPUT_TAR, DONE):
|
||||
self.store._delete(self.key(task, name))
|
||||
|
||||
def delivered(self, task: str) -> bool:
|
||||
"""Whether the runner's tar is in the store (an exact-key listing, the store's HEAD)."""
|
||||
key = self.key(task, OUTPUT_TAR)
|
||||
return key in self.store._keys(key)
|
||||
|
||||
def collect(self, task: str, outputs: Iterable[str]) -> tuple[dict[str, Any] | None, int]:
|
||||
"""Download and extract the tar under ``<base>/<task>/``, re-put the declared ``outputs``, write
|
||||
``done.json``; returns (the extracted ``summary.json`` or None, the tar's byte size)."""
|
||||
got = self.store._get(self.key(task, OUTPUT_TAR))
|
||||
if got is None:
|
||||
raise DeliveryError(f"task {task}: output.tar is not in the store")
|
||||
data = got[0]
|
||||
dest = (self.base / task).resolve()
|
||||
try:
|
||||
with tarfile.open(fileobj=io.BytesIO(data), mode="r:") as tar:
|
||||
members = tar.getmembers()
|
||||
for m in members:
|
||||
if m.issym() or m.islnk() or not (dest / m.name).resolve().is_relative_to(dest):
|
||||
raise DeliveryError(f"task {task}: rejected member {m.name}")
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
tar.extractall(dest, members) # every member checked above
|
||||
except tarfile.TarError as exc:
|
||||
raise DeliveryError(f"task {task}: bad tar: {exc}") from None
|
||||
published = self.publish(task, outputs)
|
||||
summary = _read_json(dest / "summary.json")
|
||||
done = {
|
||||
"task": task,
|
||||
"status": summary.get("status") if isinstance(summary, dict) else None,
|
||||
"bytes": len(data),
|
||||
"files": sum(1 for m in members if m.isfile()),
|
||||
"published": published,
|
||||
"finished": _now(),
|
||||
}
|
||||
self.store._put(self.key(task, DONE), json.dumps(done, sort_keys=True).encode("utf-8"))
|
||||
return (summary if isinstance(summary, dict) else None), len(data)
|
||||
|
||||
def publish(self, task: str, outputs: Iterable[str]) -> list[str]:
|
||||
"""Re-put the task's declared outputs that exist under ``<base>/<task>/`` as single objects; the names put."""
|
||||
put: list[str] = []
|
||||
root = (self.base / task).resolve()
|
||||
for artifact in outputs:
|
||||
path = (root / artifact).resolve()
|
||||
if path.is_relative_to(root) and path.is_file():
|
||||
self.store._put(self.key(task, artifact), path.read_bytes())
|
||||
put.append(artifact)
|
||||
return put
|
||||
|
||||
def put_input(self, name: str, path: Path) -> str:
|
||||
"""Store a file no task produces as ``runs/<run>/inputs/<name>``; returns its presigned GET URL."""
|
||||
self.store._put(self.key(INPUTS_DIR, name), Path(path).read_bytes())
|
||||
return self.store.presign_get(self.key(INPUTS_DIR, name), self.expires)
|
||||
|
||||
# --- state ---------------------------------------------------------------------------------------------------
|
||||
|
||||
def save_state(self, data: Mapping[str, Any]) -> None:
|
||||
self.store._put(self.key(STATE), (json.dumps(data, indent=2) + "\n").encode("utf-8"))
|
||||
|
||||
def load_state(self) -> dict[str, Any] | None:
|
||||
got = self.store._get(self.key(STATE))
|
||||
if got is None:
|
||||
return None
|
||||
try:
|
||||
loaded = json.loads(got[0].decode("utf-8"))
|
||||
except ValueError:
|
||||
return None
|
||||
return loaded if isinstance(loaded, dict) else None
|
||||
|
||||
|
||||
def _read_json(path: Path) -> Any:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
@@ -1,15 +1,35 @@
|
||||
"""``ousast plane doctor``: four checks of this host's ax before a Run is submitted (ai-service-plane Req 3.5).
|
||||
"""``ousast plane doctor``: checks of the configured cluster before a Run is submitted (ai-service-plane Req 3.5,
|
||||
plane-on-kubernetes Req 1.2).
|
||||
|
||||
Everything is read from the :class:`~openultrasast.plane.profile.PlaneProfile`: the kube context is checked for
|
||||
reachability and, when reachable, the two namespaces for ready pods; the ``images`` file's runner reference is
|
||||
resolved in its registry (a local registry over its v2 API, a remote one through ``docker manifest inspect``); the
|
||||
memory store is opened, which verifies an S3 bucket; and the first line reports every configured address. Nothing
|
||||
here names a cluster, a registry or an address of its own: those are the profile's.
|
||||
|
||||
A sibling of ``reconciler.py`` (tasks.md task 0 allows either), which re-exports :func:`doctor`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import ipaddress
|
||||
import re
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from .profile import PlaneProfile, ProfileError, load_profile
|
||||
|
||||
UP = "ops/ax/up.sh brings it up"
|
||||
RUNBOOK = "docs/deployment.md lists the prerequisites"
|
||||
_MANIFEST_TYPES = ", ".join(
|
||||
(
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --- checks ----------------------------------------------------------------------------------------------
|
||||
@@ -34,18 +54,68 @@ def _pods_ready(namespace: str, context: str) -> tuple[bool, str]:
|
||||
return not bad, f"not ready: {', '.join(bad)}" if bad else f"{len(rows)} pods ready"
|
||||
|
||||
|
||||
def doctor() -> list[tuple[str, bool, str]]:
|
||||
"""Four checks of the host's ax: kind, Agent Substrate, the ax controller, the runner image (Req 3.5)."""
|
||||
context = "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
|
||||
def _is_local(registry: str) -> bool:
|
||||
host = registry.rsplit(":", 1)[0] if re.search(r":\d+$", registry) else registry
|
||||
if host == "localhost":
|
||||
return True
|
||||
try:
|
||||
return ipaddress.ip_address(host.strip("[]")).is_loopback or ipaddress.ip_address(host.strip("[]")).is_private
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _registry_has(image: str) -> tuple[bool, str]:
|
||||
"""Resolve a digest-pinned reference ``<registry>/<repository>@sha256:<hex>``: a local registry answers its v2
|
||||
API for the manifest by digest (plain HTTP, as kind's registry speaks); a remote one is asked through
|
||||
``docker manifest inspect``, which carries the operator's registry login."""
|
||||
name, _, digest = image.partition("@")
|
||||
registry, _, repository = name.partition("/")
|
||||
short = f"{repository}@{digest[:19]}..."
|
||||
if _is_local(registry):
|
||||
url = f"http://{registry}/v2/{repository}/manifests/{digest}"
|
||||
request = urllib.request.Request(url, method="HEAD", headers={"Accept": _MANIFEST_TYPES})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=20) as response: # noqa: S310 - the profile's registry
|
||||
found = 200 <= int(response.status) < 300
|
||||
except urllib.error.HTTPError as exc:
|
||||
return False, f"{short} missing from {registry} (HTTP {exc.code})"
|
||||
except (urllib.error.URLError, OSError) as exc:
|
||||
return False, f"{registry} unreachable ({str(exc)[:80]})"
|
||||
return found, f"{short} present in {registry}" if found else f"{short} missing from {registry}"
|
||||
ok, out = _sh("docker", "manifest", "inspect", image)
|
||||
return ok, f"{short} present in {registry}" if ok else f"{short} not resolved in {registry} ({out[:120]})"
|
||||
|
||||
|
||||
def doctor(profile: PlaneProfile | None = None) -> list[tuple[str, bool, str]]:
|
||||
"""The checks of the profile's cluster: ``(name, ok, text)`` per line, the configured addresses first."""
|
||||
profile = profile or load_profile()
|
||||
hint = UP if profile.name == "kind" else RUNBOOK
|
||||
context = profile.kube_context
|
||||
checks = [(f"profile {profile.name}", True, " ".join(f"{k}={v}" for k, v in profile.addresses().items()))]
|
||||
ok, out = _sh("kubectl", "--context", context, "cluster-info")
|
||||
checks = [("kind cluster", ok, "reachable" if ok else f"context {context} unreachable ({out[:80]}); {UP}")]
|
||||
for label, ns in (("agent substrate (ate-system)", "ate-system"), ("ax controller (ax-system)", "ax-system")):
|
||||
ok, out = _pods_ready(ns, context)
|
||||
checks.append((label, ok, out if ok else f"{out}; {UP}"))
|
||||
ok, out = _sh("curl", "-s", "localhost:5001/v2/_catalog")
|
||||
present = ok and "ousast-runner" in out
|
||||
text = "ousast-runner present" if present else f"missing from localhost:5001 ({out[:80]}); {UP} and loads it"
|
||||
checks.append(("runner image in kind registry", present, text))
|
||||
checks.append((f"cluster {context}", ok, "reachable" if ok else f"context {context} unreachable ({out[:80]}); {hint}"))
|
||||
if ok: # behind an unreachable context the namespaces cannot be told apart from it
|
||||
for label, ns in (("agent substrate (ate-system)", "ate-system"), ("ax controller (ax-system)", "ax-system")):
|
||||
ok, out = _pods_ready(ns, context)
|
||||
checks.append((label, ok, out if ok else f"{out}; {hint}"))
|
||||
try:
|
||||
images = profile.load_images()
|
||||
runner = images["runner"]
|
||||
except ProfileError as exc:
|
||||
checks.append((f"runner image in {profile.registry}", False, f"{exc}; {hint}"))
|
||||
except KeyError:
|
||||
checks.append((f"runner image in {profile.registry}", False, f"{profile.images} names no runner image; {hint}"))
|
||||
else:
|
||||
ok, out = _registry_has(runner)
|
||||
checks.append((f"runner image in {profile.registry}", ok, out if ok else f"{out}; {hint} and loads it"))
|
||||
from .memory import MemoryStoreError, open_store # lazy: boto3 only when the profile names an S3 store
|
||||
|
||||
try:
|
||||
store = open_store(profile.memory)
|
||||
except (MemoryStoreError, OSError, ValueError) as exc:
|
||||
checks.append((f"memory store {profile.memory}", False, str(exc)[:300]))
|
||||
else:
|
||||
checks.append((f"memory store {profile.memory}", True, f"{store.describe()} opens (an S3 bucket is verified)"))
|
||||
return checks
|
||||
|
||||
|
||||
|
||||
@@ -84,17 +84,32 @@ def git_hosts(workspaces: Iterable[Workspace]) -> tuple[list[str], list[str]]:
|
||||
return sorted(tls), sorted(plain)
|
||||
|
||||
|
||||
def policy_for(task: Task, workspaces: Iterable[Workspace], model: Model | None, receiver_host: str | None) -> dict[str, Any]:
|
||||
"""The task's EgressPolicy body: the receiver over HTTP/80, Git and Model hosts as TLS passthrough on 443."""
|
||||
def policy_for(
|
||||
task: Task,
|
||||
workspaces: Iterable[Workspace],
|
||||
model: Model | None,
|
||||
store_host: str | None,
|
||||
receiver_host: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""The task's EgressPolicy body (plane-on-kubernetes design section 3): one ``tlsPassthrough`` rule on 443 over
|
||||
the sorted union of the store's hostname (``store_host``: where the runner PUTs its tar and GETs its inputs), the
|
||||
bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. No ``http`` rule, unless
|
||||
``receiver_host`` names the old receiver (the path task 2.6 removes). An http Git repo cannot be reached under
|
||||
this policy and is refused; a store given as an address is refused naming ``S3_ENDPOINT`` (the gateway allows
|
||||
hostnames only)."""
|
||||
tls, plain = git_hosts(workspaces)
|
||||
tls = sorted(set(tls) | set(model.egress_hosts if model else ()))
|
||||
if receiver_host and not _is_ip(receiver_host): # an address cannot be allowed; only the direct test mode has one
|
||||
plain = sorted({*plain, receiver_host.lower()})
|
||||
rules: list[dict[str, Any]] = []
|
||||
if plain:
|
||||
rules.append({"http": {"hostnames": plain, "ports": {"numbers": [80]}}})
|
||||
if tls:
|
||||
rules.append({"tlsPassthrough": {"hostnames": tls, "ports": {"numbers": [443]}}})
|
||||
raise ValueError(f"Task/{task.metadata.name}: http Git repos ({', '.join(plain)}) cannot be reached: the policy grants TLS only")
|
||||
hosts = set(tls) | set(model.egress_hosts if model else ())
|
||||
if store_host:
|
||||
if _is_ip(store_host):
|
||||
raise ValueError(f"Task/{task.metadata.name}: the store endpoint is an address ({store_host}); S3_ENDPOINT must be a hostname")
|
||||
hosts.add(store_host.lower())
|
||||
rules: list[dict[str, Any]] = []
|
||||
if receiver_host and not _is_ip(receiver_host): # the receiver path, kept for group 2 only (task 2.6 deletes it)
|
||||
rules.append({"http": {"hostnames": [receiver_host.lower()], "ports": {"numbers": [80]}}})
|
||||
if hosts:
|
||||
rules.append({"tlsPassthrough": {"hostnames": sorted(hosts), "ports": {"numbers": [443]}}})
|
||||
return {"rules": rules}
|
||||
|
||||
|
||||
@@ -110,9 +125,11 @@ def rules_of(policy: Mapping[str, Any]) -> list[dict[str, Any]]:
|
||||
return sorted(out, key=lambda r: json.dumps(r, sort_keys=True))
|
||||
|
||||
|
||||
def receiver_cluster_ip(context: str | None = None) -> str | None:
|
||||
"""The ClusterIP of the ``ousast-receiver`` Service (``kubectl get svc``); None when kubectl cannot say."""
|
||||
context = context or "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
|
||||
def receiver_cluster_ip(context: str) -> str | None:
|
||||
"""The ClusterIP of the ``ousast-receiver`` Service (``kubectl get svc`` in the profile's ``context``); None when
|
||||
kubectl cannot say."""
|
||||
if not context:
|
||||
raise RuntimeError("receiver_cluster_ip needs the profile's kube context")
|
||||
argv = ["kubectl", "--context", context, "-n", "ax-system", "get", "svc", "ousast-receiver", "-o", "jsonpath={.spec.clusterIP}"]
|
||||
try:
|
||||
proc = subprocess.run(argv, capture_output=True, text=True, check=False, timeout=20)
|
||||
@@ -122,8 +139,9 @@ def receiver_cluster_ip(context: str | None = None) -> str | None:
|
||||
return address if proc.returncode == 0 and _is_ip(address) else None
|
||||
|
||||
|
||||
def receiver_address(port: int) -> tuple[str, str | None]:
|
||||
"""(``OUSAST_ARTIFACT_URL``, ``OUSAST_ARTIFACT_DIAL``) for a receiver bound on ``port``.
|
||||
def receiver_address(port: int, context: str | None) -> tuple[str, str | None]:
|
||||
"""(``OUSAST_ARTIFACT_URL``, ``OUSAST_ARTIFACT_DIAL``) for a receiver bound on ``port``; ``context`` is the
|
||||
profile's kube context the Service's ClusterIP is looked up in.
|
||||
|
||||
``OUSAST_ARTIFACT_HOST`` set: the old direct URL ``http://<host>:<port>/`` and no dial (tests, a host the actor
|
||||
reaches without the gateway). Otherwise the receiver Service's name, dialled at ``OUSAST_ARTIFACT_DIAL`` or the
|
||||
@@ -135,7 +153,9 @@ def receiver_address(port: int) -> tuple[str, str | None]:
|
||||
return f"http://{host}:{port}/", None
|
||||
dial = os.environ.get("OUSAST_ARTIFACT_DIAL")
|
||||
if not dial:
|
||||
address = receiver_cluster_ip()
|
||||
if not context:
|
||||
raise RuntimeError("receiver_address needs the profile's kube context to look the receiver Service up")
|
||||
address = receiver_cluster_ip(context)
|
||||
dial = f"{address}:80" if address else None
|
||||
return f"http://{RECEIVER_HOST}/", dial
|
||||
|
||||
@@ -143,9 +163,11 @@ def receiver_address(port: int) -> tuple[str, str | None]:
|
||||
class Egress:
|
||||
"""``kubectl ate`` for one actor's egress policy; the executable is injectable so tests use a fake."""
|
||||
|
||||
def __init__(self, executable: str | None = None, context: str | None = None) -> None:
|
||||
def __init__(self, executable: str | None, context: str) -> None:
|
||||
if not context:
|
||||
raise ValueError("Egress needs the profile's kube context; it never builds one")
|
||||
self.executable = executable or os.environ.get("OUSAST_KUBECTL_ATE") or "kubectl-ate"
|
||||
self.context = context or "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
|
||||
self.context = context
|
||||
|
||||
def _run(self, *args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]:
|
||||
argv = [self.executable, "--context", self.context, *args]
|
||||
@@ -214,12 +236,12 @@ class Receiver(ThreadingHTTPServer):
|
||||
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, run_name: str, base: Path, port: int) -> None:
|
||||
def __init__(self, run_name: str, base: Path, port: int, context: str | None = None) -> None:
|
||||
super().__init__(("0.0.0.0", port), _Handler)
|
||||
self.run_name, self.base, self.running, self.delivered = run_name, base, set[str](), set[str]()
|
||||
self.inputs: dict[str, frozenset[str]] = {}
|
||||
self.lock = threading.Lock()
|
||||
self.url, self.dial = receiver_address(self.server_address[1])
|
||||
self.url, self.dial = receiver_address(self.server_address[1], context)
|
||||
|
||||
def _task(self, headers: Any) -> str | None:
|
||||
"""The ``X-Ousast-Task`` of a request naming this run and a task that is running now, else None."""
|
||||
|
||||
@@ -45,8 +45,9 @@ The recorded triage cost is derived per file: a batched-check record's ``usd`` p
|
||||
hunts, and its ``usage`` counts only the hunts (``verify_sinks._Recording`` slices the client's usage from the
|
||||
first hunt on), so the triage share is ``usd`` minus that usage priced at the reference model's list price.
|
||||
|
||||
``--runner-image FILE`` (``~/.cache/ousast/ax-src/runner-image``, written by ``ops/ax/up.sh``) re-pins the
|
||||
templates' ``image:`` to the digest in that file before generating, so a rebuilt image is one command.
|
||||
``--runner-image FILE`` (the profile's images file, ``ops/k8s/profiles/kind-images.json``, written by ``ops/ax/up.sh``;
|
||||
the older one-line pin file still reads) re-pins the templates' ``image:`` to the runner digest in that file before
|
||||
generating, so a rebuilt image is one command.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -82,7 +83,7 @@ TRIAGE_BASIS = (
|
||||
RANGES_BASIS = "git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')"
|
||||
HEADROOM = 1.5
|
||||
CALLS_PER_HUNT = MAX_STEPS + 2 # every tool step, the answer, one empty-content retry
|
||||
ATESPACE = "default"
|
||||
ATESPACE = "default" # the default when no profile is given; ``increment`` takes ``profile.atespace``
|
||||
CASE_PATH = "/workspace/case"
|
||||
INPUTS_PATH = "/workspace/inputs"
|
||||
FIXED_PATH = "/workspace/fixed"
|
||||
@@ -236,10 +237,10 @@ def _case_record(item: CaseInputs) -> dict[str, Any]:
|
||||
return record
|
||||
|
||||
|
||||
def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
def _workspaces(item: CaseInputs, atespace: str = ATESPACE) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
case = item.case
|
||||
git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}]
|
||||
pinned = _doc("Workspace", _ax_name(case["id"], "vulnerable"), {"git": git}, ATESPACE)
|
||||
pinned = _doc("Workspace", _ax_name(case["id"], "vulnerable"), {"git": git}, atespace)
|
||||
pinned["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={case['vulnerable']}"}
|
||||
files = {
|
||||
"candidates.json": _json({"id": case["id"], "family": case["family"], "candidates": [list(c) for c in item.kept]}),
|
||||
@@ -249,7 +250,7 @@ def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
{"candidates": [list(c) for c in item.candidates], "kept": [list(c) for c in item.kept], "triage": dict(item.triage)}
|
||||
),
|
||||
}
|
||||
inputs = _doc("Workspace", _ax_name(case["id"], "inputs"), {"files": [{"path": k, "content": v} for k, v in files.items()]}, ATESPACE)
|
||||
inputs = _doc("Workspace", _ax_name(case["id"], "inputs"), {"files": [{"path": k, "content": v} for k, v in files.items()]}, atespace)
|
||||
return pinned, inputs
|
||||
|
||||
|
||||
@@ -339,11 +340,11 @@ def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str,
|
||||
return task, entry
|
||||
|
||||
|
||||
def _alerts(item: CaseInputs, templates: Mapping[str, Task]) -> tuple[dict, dict, dict]:
|
||||
def _alerts(item: CaseInputs, templates: Mapping[str, Task], atespace: str = ATESPACE) -> tuple[dict, dict, dict]:
|
||||
"""(fixed-pin Workspace, Task, Run entry) of the case's `alerts`: quick-mode rules on both pins."""
|
||||
case = item.case
|
||||
git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}]
|
||||
fixed = _doc("Workspace", _ax_name(case["id"], "fixed"), {"git": git}, ATESPACE)
|
||||
fixed = _doc("Workspace", _ax_name(case["id"], "fixed"), {"git": git}, atespace)
|
||||
fixed["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={case['fixed']}"}
|
||||
bindings = [(_ax_name(case["id"], "vulnerable"), CASE_PATH), (fixed["metadata"]["name"], FIXED_PATH)]
|
||||
bindings.append((_ax_name(case["id"], "inputs"), INPUTS_PATH))
|
||||
@@ -358,10 +359,12 @@ def _alerts(item: CaseInputs, templates: Mapping[str, Task]) -> tuple[dict, dict
|
||||
return fixed, task, entry
|
||||
|
||||
|
||||
def _loop(cases: Sequence[CaseInputs], templates: Mapping[str, Task], loop: Loop) -> tuple[dict, list[dict], list[dict]]:
|
||||
def _loop(
|
||||
cases: Sequence[CaseInputs], templates: Mapping[str, Task], loop: Loop, atespace: str = ATESPACE
|
||||
) -> tuple[dict, list[dict], list[dict]]:
|
||||
"""(project Workspace, Tasks, Run entries) of the loop's singletons: snapshot -> measure -> propose -> improve."""
|
||||
git = [{"name": "repo", "repo": loop.repo, "dir": "repo", "depth": 1}]
|
||||
project = _doc("Workspace", _ax_name("openultrasast", loop.commit[:12]), {"git": git}, ATESPACE)
|
||||
project = _doc("Workspace", _ax_name("openultrasast", loop.commit[:12]), {"git": git}, atespace)
|
||||
project["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={loop.commit}"}
|
||||
bound = [(project["metadata"]["name"], PROJECT_PATH)]
|
||||
env = {
|
||||
@@ -399,6 +402,7 @@ def render(
|
||||
population: str = "",
|
||||
split: str = "",
|
||||
loop: Loop | None = None,
|
||||
atespace: str = ATESPACE,
|
||||
) -> dict[str, str]:
|
||||
"""Relative path under the plane root -> file text, for every generated manifest. ``population`` and ``split``
|
||||
(the population file's and the set's stems) become the Run's annotations, which the memory rows carry. Each
|
||||
@@ -411,7 +415,7 @@ def render(
|
||||
tiebreak: list[dict[str, Any]] = []
|
||||
for item in cases:
|
||||
case_id = item.case["id"]
|
||||
pinned, inputs = _workspaces(item)
|
||||
pinned, inputs = _workspaces(item, atespace)
|
||||
for doc in (pinned, inputs):
|
||||
out[f"workspaces/{doc['metadata']['name']}.yaml"] = _dump(header, [doc])
|
||||
repo = [(pinned["metadata"]["name"], CASE_PATH), (inputs["metadata"]["name"], INPUTS_PATH)]
|
||||
@@ -451,7 +455,7 @@ def render(
|
||||
entries.extend(tiebreak) # appended: the first Run's entries stay byte-identical, so a rerun keeps their state
|
||||
for item in cases: # appended after the tie-break for the same reason
|
||||
if loop is not None:
|
||||
fixed, alerts_task, alerts_entry = _alerts(item, templates)
|
||||
fixed, alerts_task, alerts_entry = _alerts(item, templates, atespace)
|
||||
out[f"workspaces/{fixed['metadata']['name']}.yaml"] = _dump(header, [fixed])
|
||||
tasks.append(alerts_task)
|
||||
entries.append(alerts_entry)
|
||||
@@ -462,7 +466,7 @@ def render(
|
||||
tasks.append(remember_task)
|
||||
entries.append(remember_entry)
|
||||
if loop is not None:
|
||||
project, loop_tasks, loop_entries = _loop(cases, templates, loop)
|
||||
project, loop_tasks, loop_entries = _loop(cases, templates, loop, atespace)
|
||||
out[f"workspaces/{project['metadata']['name']}.yaml"] = _dump(header, [project])
|
||||
tasks.extend(loop_tasks)
|
||||
entries.extend(loop_entries)
|
||||
@@ -482,11 +486,18 @@ _IMAGE_LINE = re.compile(r'^(\s*image:\s*)"?[^"\s]+"?\s*$', re.MULTILINE)
|
||||
|
||||
|
||||
def repin_templates(plane: Path, image_file: Path) -> str:
|
||||
"""Set every template's ``image:`` to the digest-pinned reference in ``image_file`` (as ``ops/ax/up.sh``
|
||||
writes it); comments and layout are kept. Returns the reference."""
|
||||
image = image_file.read_text(encoding="utf-8").strip()
|
||||
if not re.fullmatch(r"\S+@sha256:[0-9a-f]{64}", image):
|
||||
raise ValueError(f"{image_file}: expected a digest-pinned image reference, got {image!r}")
|
||||
"""Set every template's ``image:`` to the runner reference of ``image_file``: a profile images file
|
||||
(``{"runner": "<registry>/<name>@sha256:..."}``, as ``ops/ax/up.sh`` writes it) or the older one-line pin; a
|
||||
reference without a digest is refused. Comments and layout are kept. Returns the reference. (Task 4.3 gives
|
||||
the engine template ``images["engine"]``; until then only the runner key is read.)"""
|
||||
from .profile import ProfileError, load_images
|
||||
|
||||
try:
|
||||
image = load_images(image_file)["runner"]
|
||||
except ProfileError as exc:
|
||||
raise ValueError(str(exc)) from None
|
||||
except KeyError:
|
||||
raise ValueError(f"{image_file}: no runner image in the images file") from None
|
||||
for name in TEMPLATES:
|
||||
path = plane / "tasks" / f"{name}.yaml"
|
||||
text = path.read_text(encoding="utf-8")
|
||||
@@ -509,16 +520,24 @@ def increment(
|
||||
repos: Path | None = None,
|
||||
runner_image: Path | None = None,
|
||||
loop: Loop | None = None,
|
||||
atespace: str | None = None,
|
||||
) -> list[Path]:
|
||||
"""Write the generated manifests under ``plane`` from the templates in ``plane/tasks`` (re-pinned first when
|
||||
``runner_image`` is given), with the improvement loop when ``loop`` is given; returns the paths."""
|
||||
``runner_image`` is given), with the improvement loop when ``loop`` is given; returns the paths. Workspaces go
|
||||
to ``atespace`` (default: the profile's)."""
|
||||
if atespace is None:
|
||||
from .profile import load_profile
|
||||
|
||||
atespace = load_profile().atespace
|
||||
if loop is not None and repos is None:
|
||||
raise ValueError("--loop needs --repos: the alerts task marks fixed-pin alerts by the fix's new-side ranges")
|
||||
if runner_image is not None:
|
||||
repin_templates(plane, runner_image)
|
||||
loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES])
|
||||
cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos, fixed_side=loop is not None)
|
||||
files = render(cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem, loop=loop)
|
||||
files = render(
|
||||
cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem, loop=loop, atespace=atespace
|
||||
)
|
||||
written: list[Path] = []
|
||||
for relative, text in files.items():
|
||||
path = plane / relative
|
||||
|
||||
@@ -210,17 +210,19 @@ def _reader(cache: Path, source: str, ref: str, pin: str) -> Any:
|
||||
# --- manifests -------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _git_workspace(unit: Unit) -> dict[str, Any]:
|
||||
doc = _doc("Workspace", _ax_name(unit.name, "pin"), {"git": [{"name": REPO, "repo": unit.repo, "dir": REPO, "depth": 1}]}, ATESPACE)
|
||||
def _git_workspace(unit: Unit, atespace: str = ATESPACE) -> dict[str, Any]:
|
||||
doc = _doc("Workspace", _ax_name(unit.name, "pin"), {"git": [{"name": REPO, "repo": unit.repo, "dir": REPO, "depth": 1}]}, atespace)
|
||||
doc["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"{REPO}={unit.pin}"}
|
||||
return doc
|
||||
|
||||
|
||||
def _files_workspace(name: str, files: Mapping[str, str]) -> dict[str, Any]:
|
||||
return _doc("Workspace", name, {"files": [{"path": k, "content": v} for k, v in sorted(files.items())]}, ATESPACE)
|
||||
def _files_workspace(name: str, files: Mapping[str, str], atespace: str = ATESPACE) -> dict[str, Any]:
|
||||
return _doc("Workspace", name, {"files": [{"path": k, "content": v} for k, v in sorted(files.items())]}, atespace)
|
||||
|
||||
|
||||
def _verify_docs(unit: Unit, templates: Mapping[str, Task], usd_per_hunt: float) -> tuple[list[dict], list[dict], list[dict]]:
|
||||
def _verify_docs(
|
||||
unit: Unit, templates: Mapping[str, Task], usd_per_hunt: float, atespace: str = ATESPACE
|
||||
) -> tuple[list[dict], list[dict], list[dict]]:
|
||||
"""(Workspaces, Tasks, Run entries) of one verify unit."""
|
||||
asked = list(unit.verify or unit.candidates)
|
||||
candidates = {"family": unit.family, "candidates": [list(c) for c in asked]}
|
||||
@@ -232,15 +234,15 @@ def _verify_docs(unit: Unit, templates: Mapping[str, Task], usd_per_hunt: float)
|
||||
entries: list[dict] = []
|
||||
if unit.files:
|
||||
files = {f"{REPO}/{path}": text for path, text in unit.files} | {"inputs/candidates.json": _json(candidates)}
|
||||
workspace = _files_workspace(_ax_name(unit.name, "side"), files)
|
||||
workspace = _files_workspace(_ax_name(unit.name, "side"), files, atespace)
|
||||
workspaces.append(workspace)
|
||||
bindings = [(workspace["metadata"]["name"], CASE_PATH)]
|
||||
inputs = f"{CASE_PATH}/inputs/candidates.json"
|
||||
else:
|
||||
pinned = _git_workspace(unit)
|
||||
pinned = _git_workspace(unit, atespace)
|
||||
functions = [{"path": p, "function": fn, "line": ln} for p, fn, ln in asked]
|
||||
files_doc = _files_workspace(
|
||||
_ax_name(unit.name, "inputs"), {"candidates.json": _json(candidates), "functions.json": _json(functions)}
|
||||
_ax_name(unit.name, "inputs"), {"candidates.json": _json(candidates), "functions.json": _json(functions)}, atespace
|
||||
)
|
||||
workspaces += [pinned, files_doc]
|
||||
bindings = [(pinned["metadata"]["name"], CASE_PATH), (files_doc["metadata"]["name"], INPUTS_PATH)]
|
||||
@@ -309,7 +311,7 @@ def _roles_groups(units: Sequence[Unit]) -> list[list[Unit]]:
|
||||
|
||||
|
||||
def render_harvest(
|
||||
units: Sequence[Unit], templates: Mapping[str, Task], name: str, command: str, *, ceiling: float = 10.0
|
||||
units: Sequence[Unit], templates: Mapping[str, Task], name: str, command: str, *, ceiling: float = 10.0, atespace: str = ATESPACE
|
||||
) -> dict[str, str]:
|
||||
"""Relative path under the plane root -> file text: both Runs, their Workspaces and Tasks, and ``units.json``.
|
||||
A population file whose size is unknown is estimated at :data:`UNKNOWN_SIZE`."""
|
||||
@@ -323,7 +325,7 @@ def render_harvest(
|
||||
tasks: list[dict] = []
|
||||
entries: list[dict] = []
|
||||
for unit in verify_units:
|
||||
w, t, e = _verify_docs(unit, templates, USD_PER_HUNT[_kind(unit)] * scale)
|
||||
w, t, e = _verify_docs(unit, templates, USD_PER_HUNT[_kind(unit)] * scale, atespace)
|
||||
workspaces += w
|
||||
tasks += t
|
||||
entries += e
|
||||
@@ -333,7 +335,7 @@ def render_harvest(
|
||||
for group in _roles_groups([u for u in units if u.files and _inline(u, roles=True) <= MAX_INLINE_BYTES]):
|
||||
files = {f"{REPO}/{u.name}/{p}": t for u in group for p, t in u.files if p in u.paths}
|
||||
listing = sorted(f.removeprefix(f"{REPO}/") for f in files)
|
||||
workspace = _files_workspace(_ax_name(name, "roles", group[0].name), files | {"inputs/files.json": _json(listing)})
|
||||
workspace = _files_workspace(_ax_name(name, "roles", group[0].name), files | {"inputs/files.json": _json(listing)}, atespace)
|
||||
workspaces.append(workspace)
|
||||
estimates.append(_roles_estimate((len(text), len(text.splitlines())) for text in files.values()))
|
||||
bound = [(workspace["metadata"]["name"], CASE_PATH)]
|
||||
@@ -350,8 +352,8 @@ def render_harvest(
|
||||
same = [u for u in units if not u.files and (u.repo, u.pin) == (unit.repo, unit.pin)]
|
||||
paths = sorted({p for u in same for p in u.paths})
|
||||
known = {p: (chars, lines) for u in same for p, chars, lines in u.sizes}
|
||||
pinned = _git_workspace(unit)
|
||||
files_doc = _files_workspace(_ax_name(unit.name, "roles-in"), {"files.json": _json(paths)})
|
||||
pinned = _git_workspace(unit, atespace)
|
||||
files_doc = _files_workspace(_ax_name(unit.name, "roles-in"), {"files.json": _json(paths)}, atespace)
|
||||
workspaces += [w for w in (pinned, files_doc) if w["metadata"]["name"] not in emitted]
|
||||
estimates.append(_roles_estimate(known.get(p, UNKNOWN_SIZE) for p in paths))
|
||||
bound = [(pinned["metadata"]["name"], CASE_PATH), (files_doc["metadata"]["name"], INPUTS_PATH)]
|
||||
@@ -380,7 +382,16 @@ def _run_files(header: str, run: str, workspaces: list[dict], tasks: list[dict],
|
||||
|
||||
|
||||
def write_harvest(
|
||||
labels: Path, plane: Path, *, name: str, command: str, catalog: Path, cache: Path, templates: Path, ceiling: float = 10.0
|
||||
labels: Path,
|
||||
plane: Path,
|
||||
*,
|
||||
name: str,
|
||||
command: str,
|
||||
catalog: Path,
|
||||
cache: Path,
|
||||
templates: Path,
|
||||
ceiling: float = 10.0,
|
||||
atespace: str = ATESPACE,
|
||||
) -> dict[str, str]:
|
||||
"""Build the units, render, write under ``plane`` (with the Model manifests copied beside); returns the rendered
|
||||
files."""
|
||||
@@ -388,7 +399,7 @@ def write_harvest(
|
||||
|
||||
units = pair_units(labels, catalog) + population_units(labels, cache)
|
||||
loaded = load_manifests([templates / "tasks" / f"{n}.yaml" for n in ("repo-facts", "verify", "agree", "roles")]).tasks
|
||||
files = render_harvest(units, loaded, name, command, ceiling=ceiling)
|
||||
files = render_harvest(units, loaded, name, command, ceiling=ceiling, atespace=atespace)
|
||||
for path in sorted((templates / "models").glob("*.yaml")):
|
||||
files[f"models/{path.name}"] = path.read_text(encoding="utf-8")
|
||||
for relative, text in files.items():
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
"""The plane's configuration surface: one ``PlaneProfile`` per cluster (plane-on-kubernetes design section 1).
|
||||
|
||||
A profile is a TOML file under ``ops/k8s/profiles/`` (``kind.toml`` for the laptop's kind cluster, ``k3s.toml`` for
|
||||
the production cluster) holding every value that differs between clusters: the execution mode, the kube context,
|
||||
the registry and the digest-pinned images file, the router and ax-server addresses (empty: tunnel through the
|
||||
Kubernetes API), the memory store, the atespace, ax's snapshot bucket, the worker pools and the image pull secret.
|
||||
:func:`load_profile` reads the file ``OUSAST_PLANE_PROFILE`` names (a profile name or a path; default ``kind``) and
|
||||
then applies the environment: every scalar field is overridable by ``OUSAST_<FIELD>`` (``OUSAST_KUBE_CONTEXT``,
|
||||
``OUSAST_REGISTRY``, ``OUSAST_MEMORY``, ``OUSAST_ROUTER_URL``, ...), a pool by ``OUSAST_POOL_<NAME>_<FIELD>``.
|
||||
|
||||
A profile holds no secret: a key whose name ends in ``key``, ``secret``, ``token`` or ``password`` is refused, and so
|
||||
is any unknown key, an ``exec`` outside ``local``/``remote``, and an images file whose references are not pinned by
|
||||
digest (Agent Substrate rejects tags). Provider and store credentials stay in the environment or ``.env``.
|
||||
|
||||
``python -m openultrasast.plane.profile --print FIELD`` prints one value for shell scripts (``ops/ax/up.sh``):
|
||||
``registry``, ``kube_context``, ``images.runner``, ``pools.default.replicas``, ...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field, fields
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
__all__ = ["DEFAULT_PROFILE", "PROFILES_DIR", "PlaneProfile", "Pool", "ProfileError", "load_images", "load_profile", "profiles_dir"]
|
||||
|
||||
DEFAULT_PROFILE = "kind"
|
||||
PROFILES_DIR = Path("ops/k8s/profiles")
|
||||
EXEC_MODES = ("local", "remote")
|
||||
_DIGEST = re.compile(r"\S+@sha256:[0-9a-f]{64}")
|
||||
_SECRET_KEY = re.compile(r"(key|secret|token|password)$", re.IGNORECASE)
|
||||
_SECRET_VALUE = re.compile(r"(sk-[A-Za-z0-9]{8,}|AKIA[0-9A-Z]{12,}|[A-Za-z0-9+/]{40,}={0,2}$)")
|
||||
_ENV_PREFIX = "OUSAST_"
|
||||
|
||||
|
||||
class ProfileError(ValueError):
|
||||
"""A profile that cannot be used; the message names the file and the field."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pool:
|
||||
"""One WorkerPool's size: how many warm workers, and each worker's CPU and memory limit."""
|
||||
|
||||
replicas: int
|
||||
cpu: str
|
||||
memory: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlaneProfile:
|
||||
"""The values of one cluster. ``images`` is resolved against the profile file's directory."""
|
||||
|
||||
name: str
|
||||
path: Path
|
||||
exec: str = "local"
|
||||
kube_context: str = ""
|
||||
registry: str = ""
|
||||
images: Path = Path("images.json")
|
||||
router_url: str = ""
|
||||
ax_server: str = ""
|
||||
memory: str = ""
|
||||
atespace: str = "default"
|
||||
snapshots_bucket: str = ""
|
||||
image_pull_secret: str = ""
|
||||
pools: Mapping[str, Pool] = field(default_factory=dict)
|
||||
|
||||
def load_images(self) -> dict[str, str]:
|
||||
"""The digest-pinned image references of ``images`` (``{"runner": ..., "engine": ...}``)."""
|
||||
return load_images(self.images)
|
||||
|
||||
def addresses(self) -> dict[str, str]:
|
||||
"""Every configured address, for ``ousast plane doctor`` (Req 1.2); an empty router or ax-server address
|
||||
means the Kubernetes API tunnel."""
|
||||
return {
|
||||
"exec": self.exec,
|
||||
"kube_context": self.kube_context,
|
||||
"registry": self.registry,
|
||||
"images": str(self.images),
|
||||
"router_url": self.router_url or "(tunnel through the kube context)",
|
||||
"ax_server": self.ax_server or "(tunnel through the kube context)",
|
||||
"memory": self.memory,
|
||||
"atespace": self.atespace,
|
||||
"snapshots_bucket": self.snapshots_bucket,
|
||||
"image_pull_secret": self.image_pull_secret or "(none)",
|
||||
}
|
||||
|
||||
|
||||
_SCALARS = {f.name: str(f.type) for f in fields(PlaneProfile) if f.name not in ("name", "path", "pools")}
|
||||
_POOL_FIELDS = {f.name: str(f.type) for f in fields(Pool)}
|
||||
|
||||
|
||||
def profiles_dir() -> Path:
|
||||
"""``ops/k8s/profiles`` of this checkout (``OUSAST_PLANE_PROFILES`` overrides; an installed package falls back
|
||||
to the working directory)."""
|
||||
override = os.environ.get("OUSAST_PLANE_PROFILES")
|
||||
if override:
|
||||
return Path(override)
|
||||
checkout = Path(__file__).resolve().parents[3] / PROFILES_DIR
|
||||
return checkout if checkout.is_dir() else PROFILES_DIR
|
||||
|
||||
|
||||
def load_images(path: Path) -> dict[str, str]:
|
||||
"""``{"runner": "<registry>/<name>@sha256:<64 hex>", ...}`` from a JSON images file; every value must be
|
||||
digest-pinned. A one-line file holding a single reference (``ops/ax/up.sh`` before this profile) reads as
|
||||
``{"runner": <line>}``."""
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
raise ProfileError(f"images file {path}: {exc.strerror or exc}") from None
|
||||
try:
|
||||
loaded: Any = json.loads(text) if text.startswith("{") else {"runner": text}
|
||||
except ValueError as exc:
|
||||
raise ProfileError(f"images file {path}: not JSON ({exc})") from None
|
||||
if not isinstance(loaded, dict) or not loaded or not all(isinstance(k, str) and isinstance(v, str) for k, v in loaded.items()):
|
||||
raise ProfileError(f"images file {path}: expected an object of image names to references")
|
||||
for name, reference in loaded.items():
|
||||
if not _DIGEST.fullmatch(reference):
|
||||
raise ProfileError(f"images file {path}: {name} is not digest-pinned ({reference!r}); Agent Substrate rejects tags")
|
||||
return dict(loaded)
|
||||
|
||||
|
||||
def _reject_secret(where: str, key: str, value: object, declared: bool = False) -> None:
|
||||
"""A key that is not a declared field and is named like a credential, or any value shaped like one, is refused.
|
||||
``image_pull_secret`` is declared: its value is the *name* of a Kubernetes Secret, never its content."""
|
||||
if not declared and _SECRET_KEY.search(key):
|
||||
raise ProfileError(f"{where}: {key} looks like a credential; profiles hold no secret (use the environment or .env)")
|
||||
if isinstance(value, str) and _SECRET_VALUE.search(value):
|
||||
raise ProfileError(f"{where}: the value of {key} looks like a credential; profiles hold no secret")
|
||||
|
||||
|
||||
def _coerce(where: str, key: str, value: object, kind: str) -> Any:
|
||||
if kind == "int":
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ProfileError(f"{where}: {key} must be an integer, got {value!r}")
|
||||
return value
|
||||
if kind == "Path":
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ProfileError(f"{where}: {key} must be a file name")
|
||||
return Path(value)
|
||||
if not isinstance(value, str):
|
||||
raise ProfileError(f"{where}: {key} must be a string, got {value!r}")
|
||||
return value
|
||||
|
||||
|
||||
def _pools(where: str, raw: object) -> dict[str, Pool]:
|
||||
if not isinstance(raw, dict):
|
||||
raise ProfileError(f"{where}: pools must be a table of pool names")
|
||||
out: dict[str, Pool] = {}
|
||||
for name, spec in raw.items():
|
||||
if not isinstance(spec, dict):
|
||||
raise ProfileError(f"{where}: pools.{name} must be a table")
|
||||
unknown = sorted(set(spec) - set(_POOL_FIELDS))
|
||||
if unknown:
|
||||
raise ProfileError(f"{where}: pools.{name} has unknown keys: {', '.join(unknown)}")
|
||||
missing = sorted(set(_POOL_FIELDS) - set(spec))
|
||||
if missing:
|
||||
raise ProfileError(f"{where}: pools.{name} lacks {', '.join(missing)}")
|
||||
for key, value in spec.items():
|
||||
_reject_secret(where, f"pools.{name}.{key}", value, declared=True)
|
||||
out[str(name)] = Pool(**{k: _coerce(where, f"pools.{name}.{k}", spec[k], _POOL_FIELDS[k]) for k in _POOL_FIELDS})
|
||||
return out
|
||||
|
||||
|
||||
def _env_overrides(environ: Mapping[str, str], where: str, values: dict[str, Any], pools: dict[str, Pool]) -> None:
|
||||
for name, kind in _SCALARS.items():
|
||||
raw = environ.get(_ENV_PREFIX + name.upper())
|
||||
if raw is not None and raw != "":
|
||||
value: Any = raw
|
||||
if kind == "int":
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
raise ProfileError(f"{where}: {_ENV_PREFIX}{name.upper()} must be an integer, got {raw!r}") from None
|
||||
values[name] = _coerce(where, _ENV_PREFIX + name.upper(), value, kind)
|
||||
pattern = re.compile(rf"^{_ENV_PREFIX}POOL_([A-Z0-9]+)_(REPLICAS|CPU|MEMORY)$")
|
||||
for variable, raw in environ.items():
|
||||
found = pattern.match(variable)
|
||||
if not found or raw == "":
|
||||
continue
|
||||
pool_name, key = found.group(1).lower(), found.group(2).lower()
|
||||
current = pools.get(pool_name)
|
||||
if current is None:
|
||||
raise ProfileError(f"{where}: {variable} names a pool the profile does not declare ({pool_name})")
|
||||
value = int(raw) if key == "replicas" and raw.isdigit() else raw
|
||||
pools[pool_name] = Pool(**{**vars(current), key: _coerce(where, variable, value, _POOL_FIELDS[key])})
|
||||
|
||||
|
||||
def load_profile(name: str | None = None, environ: Mapping[str, str] | None = None) -> PlaneProfile:
|
||||
"""The profile ``name`` (a name under :func:`profiles_dir` or a path to a ``.toml``; default
|
||||
``OUSAST_PLANE_PROFILE``, else ``kind``), with the environment's overrides applied; :class:`ProfileError`
|
||||
names what is wrong."""
|
||||
env = os.environ if environ is None else environ
|
||||
chosen = name or env.get("OUSAST_PLANE_PROFILE") or DEFAULT_PROFILE
|
||||
path = Path(chosen) if chosen.endswith(".toml") or "/" in chosen else profiles_dir() / f"{chosen}.toml"
|
||||
where = str(path)
|
||||
try:
|
||||
raw = tomllib.loads(path.read_text(encoding="utf-8"))
|
||||
except OSError as exc:
|
||||
raise ProfileError(f"profile {where}: {exc.strerror or exc}") from None
|
||||
except tomllib.TOMLDecodeError as exc:
|
||||
raise ProfileError(f"profile {where}: {exc}") from None
|
||||
for key, value in raw.items(): # a credential-shaped key is named as such before it is merely unknown
|
||||
_reject_secret(where, key, value, declared=key in _SCALARS)
|
||||
unknown = sorted(set(raw) - set(_SCALARS) - {"pools"})
|
||||
if unknown:
|
||||
raise ProfileError(f"{where}: unknown keys: {', '.join(unknown)}")
|
||||
values: dict[str, Any] = {}
|
||||
for key, value in raw.items():
|
||||
if key != "pools":
|
||||
values[key] = _coerce(where, key, value, _SCALARS[key])
|
||||
pools = _pools(where, raw.get("pools", {}))
|
||||
_env_overrides(env, where, values, pools)
|
||||
if values.get("exec", "local") not in EXEC_MODES:
|
||||
raise ProfileError(f"{where}: exec must be one of {', '.join(EXEC_MODES)}, got {values.get('exec')!r}")
|
||||
images = Path(values.get("images", "images.json"))
|
||||
if not images.is_absolute():
|
||||
images = path.resolve().parent / images
|
||||
values["images"] = images
|
||||
if images.is_file():
|
||||
load_images(images) # a committed file must be pinned; a file the release has not published yet is doctor's finding
|
||||
return PlaneProfile(name=path.stem, path=path, pools=pools, **values)
|
||||
|
||||
|
||||
def _lookup(profile: PlaneProfile, dotted: str) -> str:
|
||||
head, _, rest = dotted.partition(".")
|
||||
if head == "images" and rest:
|
||||
images = profile.load_images()
|
||||
if rest not in images:
|
||||
raise ProfileError(f"{profile.images}: no image named {rest!r} (have {', '.join(sorted(images))})")
|
||||
return images[rest]
|
||||
if head == "pools":
|
||||
pool_name, _, key = rest.partition(".")
|
||||
if pool_name not in profile.pools or key not in _POOL_FIELDS:
|
||||
raise ProfileError(f"{profile.path}: no such pool field {dotted!r}")
|
||||
return str(getattr(profile.pools[pool_name], key))
|
||||
if head not in _SCALARS or rest:
|
||||
raise ProfileError(f"{profile.path}: no such field {dotted!r}")
|
||||
return str(getattr(profile, head))
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""``python -m openultrasast.plane.profile [--profile NAME] --print FIELD [FIELD...]``: one value per line."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(prog="openultrasast.plane.profile", description=main.__doc__)
|
||||
parser.add_argument("--profile", help="profile name or path (default OUSAST_PLANE_PROFILE, else kind)")
|
||||
parser.add_argument("--print", dest="fields", nargs="+", required=True, metavar="FIELD")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
profile = load_profile(args.profile)
|
||||
for dotted in args.fields:
|
||||
print(_lookup(profile, dotted))
|
||||
except ProfileError as exc:
|
||||
print(f"profile: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,23 +1,26 @@
|
||||
"""ax-backed reconciler of a ``Run``: submission, resume, start, completion by delivery (ai-service-plane Req 3, 4, 7).
|
||||
|
||||
ax executes every task on this host's kind cluster; nothing here runs a task locally or knows a prompt, a score or
|
||||
a model call. Per ready task: one rendered YAML file (the Task with its env extended, its Workspaces, the bound
|
||||
Model), ``ax apply -f``, ``ax resume task`` (retried on DeadlineExceeded/Unavailable -- a new image's first resume
|
||||
waits for Agent Substrate's golden snapshot -- for up to ``OUSAST_RESUME_TIMEOUT`` s, 900), and once ax says
|
||||
ax executes every task on the profile's cluster (``profile.py``: kube context, registry, store); nothing here runs a
|
||||
task locally or knows a prompt, a score or a model call. Per ready task: one rendered YAML file (the Task with its
|
||||
env extended, its Workspaces, the bound Model), ``ax apply -f``, ``ax resume task`` (retried on
|
||||
DeadlineExceeded/Unavailable -- a new image's first resume waits for Agent Substrate's golden snapshot -- for up to
|
||||
``OUSAST_RESUME_TIMEOUT`` s, 900), and once ax says
|
||||
``Running`` one start request through Substrate's router (``router.py``, Req 4.5) carrying the bound Model's
|
||||
credential from this process's environment, never from a file (Req 4.6). Completion is the runner's artifact
|
||||
delivery (ax has no Completed phase); ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s
|
||||
(7200) without delivery fail it; ``ax delete task`` follows.
|
||||
credential from this process's environment, never from a file (Req 4.6), and the presigned store URLs of
|
||||
``delivery.py`` (plane-on-kubernetes design section 3). Completion is the runner's output tar appearing in the
|
||||
memory store (ax has no Completed phase), which the reconciler polls, downloads, extracts and re-publishes per
|
||||
declared output; ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without
|
||||
delivery fail it; ``ax delete task`` follows. ``OUSAST_DELIVERY=receiver`` keeps the host receiver of the kind
|
||||
profile for group 2 only (task 2.6 removes it).
|
||||
|
||||
ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the
|
||||
task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's pin annotation as
|
||||
``OUSAST_GIT_PINS``. Inputs other tasks produced are never rendered: ``OUSAST_INPUTS`` maps each input's name to
|
||||
``<producer>/<artifact>``, which the runner fetches after the start from the receiver's ``GET /inputs/...`` to
|
||||
``OUSAST_INPUT_<NAME>`` under ``/workspace/.ousast-in/<task>`` (the receiver serves a running task its declared
|
||||
inputs only). The receiver takes the runner's tar (``X-Ousast-Run``, ``X-Ousast-Task``) on
|
||||
``OUSAST_ARTIFACT_PORT`` (18090), reached by its Service name through the egress gateway; between ``ax apply`` and
|
||||
resume the task's egress policy is written (``egress.py``).
|
||||
State: ``~/ousast-results/plane/<run>/`` (``OUSAST_RESULTS``), polled every ``OUSAST_POLL_SECONDS`` (3).
|
||||
``<producer>/<artifact>``, fetched after the start by the presigned GET of the start body to ``OUSAST_INPUT_<NAME>``
|
||||
under ``/workspace/.ousast-in/<task>``. Between ``ax apply`` and resume the task's egress policy is written
|
||||
(``egress.py``). State: ``~/ousast-results/plane/<run>/`` (``OUSAST_RESULTS``), polled every ``OUSAST_POLL_SECONDS``
|
||||
(3), mirrored to ``runs/<run>/state.json`` on the store after every change and seeded from there by a rerun with no
|
||||
local state (design section 2); the store is opened first and required, there is no fallback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -41,9 +44,12 @@ from urllib.parse import urlsplit
|
||||
|
||||
import yaml
|
||||
|
||||
from .delivery import Delivery, DeliveryError, declared
|
||||
from .doctor import doctor
|
||||
from .egress import RECEIVER_PORT, Egress, Receiver, policy_for
|
||||
from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests
|
||||
from .memory import open_store
|
||||
from .profile import PlaneProfile, load_profile
|
||||
from .router import Router, credentials, open_router
|
||||
|
||||
MODEL_ANNOTATION = "openultrasast.io/model"
|
||||
@@ -136,15 +142,16 @@ def _model_doc(model: Model) -> dict[str, Any]:
|
||||
return _doc("Model", model.metadata.name, spec, model.metadata.atespace)
|
||||
|
||||
|
||||
def render_task(run: Run, entry: RunTask, manifests: Manifests, artifact_url: str, dial: str | None = None) -> list[dict[str, Any]]:
|
||||
def render_task(run: Run, entry: RunTask, manifests: Manifests, artifact_url: str = "", dial: str | None = None) -> list[dict[str, Any]]:
|
||||
"""The documents ``ax apply`` receives for one Run task: its Model, its Workspaces, the Task last. Task-produced
|
||||
inputs are never rendered: ax puts every bound Workspace into one env value, which Substrate caps at 32768
|
||||
characters, so the runner fetches them from the receiver (``OUSAST_INPUTS``) after the start."""
|
||||
inputs are never rendered (Substrate caps one env value at 32768 characters): the runner fetches them
|
||||
(``OUSAST_INPUTS``) after the start by presigned URL, or from the receiver when ``artifact_url`` is given."""
|
||||
task: Task = manifests.tasks[entry.task]
|
||||
ax_name = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT)
|
||||
env = {e.name: e.value for e in task.env}
|
||||
env.update(OUSAST_RUN=run.metadata.name, OUSAST_TASK=entry.name, OUSAST_OUTPUT_DIR=f"{OUTPUT_ROOT}/{entry.name}")
|
||||
env.update(OUSAST_ARTIFACT_URL=artifact_url, **({"OUSAST_ARTIFACT_DIAL": dial} if dial else {}))
|
||||
if artifact_url: # receiver mode only (task 2.6 removes it): the store path carries no URL in the env
|
||||
env.update(OUSAST_ARTIFACT_URL=artifact_url, **({"OUSAST_ARTIFACT_DIAL": dial} if dial else {}))
|
||||
budget = _fields(entry.budget) if entry.budget else {}
|
||||
env.update({f"OUSAST_BUDGET_{k.upper()}": str(v) for k, v in budget.items()})
|
||||
inputs = {re.sub(r"[^A-Z0-9]+", "_", name.upper()): ref for name, ref in entry.inputs.items()}
|
||||
@@ -224,12 +231,25 @@ class _State:
|
||||
path: Path
|
||||
data: dict[str, Any]
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
mirror: Callable[[Mapping[str, Any]], None] | None = None # the store copy; a failure is reported, never fatal
|
||||
|
||||
def set(self, task: str, **fields: object) -> None:
|
||||
with self.lock:
|
||||
self.data["tasks"].setdefault(task, {"status": "pending", "started": None, "finished": None, "usd": None, "model": None})
|
||||
self.data["tasks"][task].update(fields)
|
||||
_write_json(self.path, self.data)
|
||||
self._flush()
|
||||
|
||||
def flush(self) -> None:
|
||||
with self.lock:
|
||||
self._flush()
|
||||
|
||||
def _flush(self) -> None:
|
||||
_write_json(self.path, self.data)
|
||||
if self.mirror is not None:
|
||||
try:
|
||||
self.mirror(self.data)
|
||||
except Exception as exc: # noqa: BLE001 - the local file is written; the mirror is reported, the run goes on
|
||||
print(f"state mirror to the store failed: {exc}", file=sys.stderr)
|
||||
|
||||
def status(self, task: str) -> str:
|
||||
return str(self.data["tasks"].get(task, {}).get("status", "pending"))
|
||||
@@ -313,25 +333,46 @@ def _await(ax: Ax, name: str, delivered: Callable[[], bool], start: Callable[[],
|
||||
|
||||
|
||||
def _execute(
|
||||
ax: Ax, egress: Egress, run: Run, entry: RunTask, manifests: Manifests, base: Path, receiver: Receiver, router: Router
|
||||
ax: Ax,
|
||||
egress: Egress,
|
||||
run: Run,
|
||||
entry: RunTask,
|
||||
manifests: Manifests,
|
||||
base: Path,
|
||||
delivery: Delivery,
|
||||
router: Router,
|
||||
receiver: Receiver | None = None,
|
||||
) -> tuple[str, str]:
|
||||
ax_name, task = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT), manifests.tasks[entry.task]
|
||||
atespace, manifest = task.metadata.atespace or "default", base / entry.name / "task.yaml"
|
||||
manifest.parent.mkdir(parents=True, exist_ok=True)
|
||||
docs = render_task(run, entry, manifests, receiver.url, receiver.dial)
|
||||
docs = render_task(run, entry, manifests, receiver.url if receiver else "", receiver.dial if receiver else None)
|
||||
manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8")
|
||||
model = manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "")
|
||||
policy = policy_for(task, [manifests.workspaces[b.name] for b in task.workspaces], model, urlsplit(receiver.url).hostname)
|
||||
workspaces = [manifests.workspaces[b.name] for b in task.workspaces]
|
||||
body = None if receiver else delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body
|
||||
store_host, receiver_host = (None, urlsplit(receiver.url).hostname) if receiver else (delivery.host, None)
|
||||
if receiver is None:
|
||||
delivery.reset(entry.name) # a previous attempt's tar must not complete this one
|
||||
|
||||
def delivered() -> bool:
|
||||
return entry.name in receiver.delivered if receiver is not None else delivery.delivered(entry.name)
|
||||
|
||||
try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request
|
||||
start = router.starter(atespace, ax_name, run.metadata.name, credentials(model))
|
||||
policy = policy_for(task, workspaces, model, store_host, receiver_host)
|
||||
start = router.starter(atespace, ax_name, run.metadata.name, credentials(model), body)
|
||||
ax.apply(manifest) # creates the Substrate actor; its egress is denied until the policy exists
|
||||
egress.apply(ax_name, atespace, policy)
|
||||
egress.settle()
|
||||
except RuntimeError as exc: # StartError included: a Model-bound task whose secret is unset
|
||||
except (RuntimeError, ValueError) as exc: # StartError (a Model's secret unset) and a policy that cannot be granted
|
||||
ax.delete(ax_name)
|
||||
return "failed", str(exc)
|
||||
try:
|
||||
failure = _await(ax, ax_name, lambda: entry.name in receiver.delivered, start)
|
||||
failure = _await(ax, ax_name, delivered, start)
|
||||
if failure is None and receiver is None:
|
||||
delivery.collect(entry.name, declared(run)[entry.name]) # download, extract, re-publish what is declared
|
||||
except DeliveryError as exc:
|
||||
failure = str(exc)
|
||||
finally:
|
||||
ax.delete(ax_name)
|
||||
leftover = egress.delete(ax_name, atespace) # the policy goes with its actor; a survivor is reported
|
||||
@@ -340,20 +381,44 @@ def _execute(
|
||||
return ("failed", failure) if failure else outcome_of(_read_json(base / entry.name / "summary.json"))
|
||||
|
||||
|
||||
def run(run_manifest: Path, *, workers: int = 1, ax: str = "ax", results_root: Path | None = None, kubectl_ate: str | None = None) -> str:
|
||||
"""Execute a Run on ax and return its status: ``done``, ``unfinished`` or ``failed`` (Req 3.1-3.3, 2.2)."""
|
||||
def run(
|
||||
run_manifest: Path,
|
||||
*,
|
||||
workers: int = 1,
|
||||
ax: str = "ax",
|
||||
results_root: Path | None = None,
|
||||
kubectl_ate: str | None = None,
|
||||
profile: PlaneProfile | None = None,
|
||||
) -> str:
|
||||
"""Execute a Run on ax and return its status: ``done``, ``unfinished`` or ``failed`` (Req 3.1-3.3, 2.2). Every
|
||||
address comes from ``profile`` (default: the one ``OUSAST_PLANE_PROFILE`` names)."""
|
||||
profile = profile or load_profile()
|
||||
run_spec, manifests = load_run(Path(run_manifest))
|
||||
base = run_dir(run_spec.metadata.name, results_root)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
store = open_store(profile.memory) # first, before the lock and before ax sees anything: no fallback
|
||||
delivery = Delivery(store, run_spec.metadata.name, base)
|
||||
if not (base / "state.json").exists():
|
||||
seeded = delivery.load_state() # a rerun from another machine or an empty results root
|
||||
if seeded is not None:
|
||||
_write_json(base / "state.json", seeded)
|
||||
lock = _acquire_lock(base)
|
||||
state = _State(base / "state.json", _read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}})
|
||||
port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT)
|
||||
receiver = Receiver(run_spec.metadata.name, base, int(port)) # the receiver Service targets RECEIVER_PORT
|
||||
threading.Thread(target=receiver.serve_forever, daemon=True).start()
|
||||
cli, egress, finished = Ax(ax), Egress(kubectl_ate), queue.Queue[tuple[str, str, str]]()
|
||||
state = _State(
|
||||
base / "state.json",
|
||||
_read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}},
|
||||
mirror=delivery.save_state,
|
||||
)
|
||||
receiver: Receiver | None = None
|
||||
if os.environ.get("OUSAST_DELIVERY") == "receiver": # the kind profile's host receiver, group 2 only (2.6 removes it)
|
||||
port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT)
|
||||
receiver = Receiver(run_spec.metadata.name, base, int(port), profile.kube_context) # the Service targets RECEIVER_PORT
|
||||
threading.Thread(target=receiver.serve_forever, daemon=True).start()
|
||||
cli, egress, finished = Ax(ax), Egress(kubectl_ate, profile.kube_context), queue.Queue[tuple[str, str, str]]()
|
||||
pending = [t for t in run_spec.tasks if state.status(t.name) != "done"]
|
||||
for entry in run_spec.tasks:
|
||||
state.set(entry.name, **({"status": "pending"} if entry in pending else {}))
|
||||
if entry not in pending:
|
||||
delivery.publish(entry.name, declared(run_spec)[entry.name]) # a seeded or earlier task's artifacts reach its consumers
|
||||
running: dict[str, RunTask] = {}
|
||||
halted = False
|
||||
|
||||
@@ -361,10 +426,14 @@ def run(run_manifest: Path, *, workers: int = 1, ax: str = "ax", results_root: P
|
||||
if state.status(entry.name) in ("running", "suspended"):
|
||||
cli.delete(_ax_name(run_spec.metadata.name, entry.name, limit=TASK_NAME_LIMIT)) # an interrupted attempt; ax must not keep it
|
||||
state.set(entry.name, status="running", started=_now(), finished=None)
|
||||
finished.put((entry.name, *_execute(cli, egress, run_spec, entry, manifests, base, receiver, router)))
|
||||
try:
|
||||
outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router, receiver)
|
||||
except Exception as exc: # noqa: BLE001 - a worker that dies silently would hang the run on the queue
|
||||
outcome = ("failed", f"{type(exc).__name__}: {exc}")
|
||||
finished.put((entry.name, *outcome))
|
||||
|
||||
try:
|
||||
with open_router() as router: # one port-forward to Substrate's router for the whole run
|
||||
with open_router(profile.kube_context, url=profile.router_url or None) as router: # one tunnel per run (ax-tunnel)
|
||||
while pending or running:
|
||||
blocked = {t.serialize for t in running.values() if t.serialize}
|
||||
for entry in list(pending):
|
||||
@@ -375,29 +444,32 @@ def run(run_manifest: Path, *, workers: int = 1, ax: str = "ax", results_root: P
|
||||
pending.remove(entry)
|
||||
running[entry.name] = entry
|
||||
blocked.add(entry.serialize or "")
|
||||
with receiver.lock: # the receiver serves a running task its declared inputs, nothing else
|
||||
receiver.running.add(entry.name)
|
||||
receiver.inputs[entry.name] = frozenset(entry.inputs.values())
|
||||
if receiver is not None:
|
||||
with receiver.lock: # the receiver serves a running task its declared inputs, nothing else
|
||||
receiver.running.add(entry.name)
|
||||
receiver.inputs[entry.name] = frozenset(entry.inputs.values())
|
||||
threading.Thread(target=worker, args=(entry,), daemon=True).start()
|
||||
if not running:
|
||||
break # nothing can start: a producer stopped short
|
||||
name, status, reason = finished.get()
|
||||
del running[name]
|
||||
with receiver.lock:
|
||||
receiver.running.discard(name)
|
||||
receiver.inputs.pop(name, None)
|
||||
if receiver is not None:
|
||||
with receiver.lock:
|
||||
receiver.running.discard(name)
|
||||
receiver.inputs.pop(name, None)
|
||||
summary = _read_json(base / name / "summary.json") or {}
|
||||
state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None)
|
||||
halted = halted or status == "failed"
|
||||
if status == "unfinished":
|
||||
pending = [t for t in pending if name not in t.producers]
|
||||
finally:
|
||||
receiver.shutdown()
|
||||
if receiver is not None:
|
||||
receiver.shutdown()
|
||||
lock.unlink(missing_ok=True)
|
||||
statuses = [state.status(t.name) for t in run_spec.tasks]
|
||||
result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished"
|
||||
state.data.update(status=result, finished=_now())
|
||||
_write_json(state.path, state.data)
|
||||
state.flush()
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping
|
||||
from typing import Any
|
||||
|
||||
from .manifests import Model
|
||||
|
||||
@@ -77,7 +78,9 @@ class Router:
|
||||
def __init__(self, url: str) -> None:
|
||||
self.url = url.rstrip("/")
|
||||
|
||||
def starter(self, atespace: str, task: str, run: str, credentials: Mapping[str, str]) -> Callable[[], str | None]:
|
||||
def starter(
|
||||
self, atespace: str, task: str, run: str, credentials: Mapping[str, str], delivery: Mapping[str, Any] | None = None
|
||||
) -> Callable[[], str | None]:
|
||||
""":meth:`start_task` bound to one task, returning the refusal text instead of raising (None once accepted).
|
||||
|
||||
The reconciler calls it again after a re-resume, since a resumed actor may have rebooted and wait for a
|
||||
@@ -90,20 +93,30 @@ class Router:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
try:
|
||||
self.start_task(atespace, task, run, credentials)
|
||||
self.start_task(atespace, task, run, credentials, delivery)
|
||||
except StartError as exc:
|
||||
return None if calls > 1 and ALREADY_STARTED in str(exc) else str(exc)
|
||||
return None
|
||||
|
||||
return start
|
||||
|
||||
def start_task(self, atespace: str, task: str, run: str, credentials: Mapping[str, str]) -> None:
|
||||
"""Start ``<atespace>/<task>``'s command once; :class:`StartError` on 409, another 4xx, or the timeout."""
|
||||
body = json.dumps({"run": run, "task": task, "credentials": dict(credentials)}).encode("utf-8")
|
||||
def start_task(
|
||||
self, atespace: str, task: str, run: str, credentials: Mapping[str, str], delivery: Mapping[str, Any] | None = None
|
||||
) -> None:
|
||||
"""Start ``<atespace>/<task>``'s command once; :class:`StartError` on 409, another 4xx, or the timeout.
|
||||
``delivery`` (``{put, inputs}`` presigned URLs, ``delivery.py``) rides in the body like the credentials and is
|
||||
redacted like them."""
|
||||
payload: dict[str, Any] = {"run": run, "task": task, "credentials": dict(credentials)}
|
||||
if delivery:
|
||||
payload["delivery"] = dict(delivery)
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
headers = {"Content-Type": "application/json", TARGET_HEADER: f"{atespace}/{task}"}
|
||||
limit = float(os.environ.get("OUSAST_START_TIMEOUT") or 300)
|
||||
deadline, backoff, last = time.monotonic() + limit, 0.5, "no attempt made"
|
||||
secrets = list(credentials.values())
|
||||
if delivery:
|
||||
secrets.append(str(delivery.get("put") or ""))
|
||||
secrets.extend(str(u) for u in (delivery.get("inputs") or {}).values())
|
||||
while True:
|
||||
request = urllib.request.Request(self.url + START_PATH, data=body, method="POST", headers=headers)
|
||||
try:
|
||||
@@ -131,18 +144,20 @@ def _drain(stream: Iterable[str], lines: queue.Queue[str | None]) -> None:
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def open_router(context: str | None = None, *, kubectl: str = "kubectl", ready_timeout: float = 30.0) -> Iterator[Router]:
|
||||
def open_router(context: str, *, url: str | None = None, kubectl: str = "kubectl", ready_timeout: float = 30.0) -> Iterator[Router]:
|
||||
"""A :class:`Router` for the duration of the block.
|
||||
|
||||
``OUSAST_ROUTER_URL`` set: that URL, nothing started. Otherwise ``kubectl port-forward svc/atenet-router :80``
|
||||
in ``ate-system`` of ``context`` (default ``kind-$KIND_CLUSTER_NAME``, ``kind-ousast``), the local port read
|
||||
from kubectl's ``Forwarding from 127.0.0.1:NNNN`` line; on exit that child process is terminated.
|
||||
``url`` (the profile's ``router_url``) or ``OUSAST_ROUTER_URL`` set: that URL, nothing started. Otherwise
|
||||
``kubectl port-forward svc/atenet-router :80`` in ``ate-system`` of the profile's ``context`` (ax-tunnel: the
|
||||
Kubernetes API is the one authenticated door), the local port read from kubectl's ``Forwarding from
|
||||
127.0.0.1:NNNN`` line; on exit that child process is terminated. No context is ever built here.
|
||||
"""
|
||||
override = os.environ.get("OUSAST_ROUTER_URL")
|
||||
override = url or os.environ.get("OUSAST_ROUTER_URL")
|
||||
if override:
|
||||
yield Router(override)
|
||||
return
|
||||
context = context or "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
|
||||
if not context:
|
||||
raise StartError("open_router needs the profile's kube context (or router_url)")
|
||||
argv = [kubectl, "--context", context, "-n", NAMESPACE, "port-forward", SERVICE, ":80"]
|
||||
proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
try:
|
||||
|
||||
@@ -10,18 +10,23 @@ not (ax docs/runner.md).
|
||||
The command never starts by itself (Requirements 4.5, 4.6). Agent Substrate boots every new ActorTemplate once as a
|
||||
"golden" actor to snapshot it, with this image and the same ``AX_TASK_YAML``/``AX_WORKSPACES_YAML``/env, and inside
|
||||
the sandbox nothing tells that boot from the real one. So after the workspaces are ready the runner waits for
|
||||
``POST /ousast/v1/start`` with ``{"run", "task", "credentials"}``, which the reconciler sends through Substrate's
|
||||
``POST /ousast/v1/start`` with ``{"run", "task", "credentials", "delivery"}``, which the reconciler sends through Substrate's
|
||||
router to the task's actor only (``plane/router.py``). ``run`` and ``task`` must match ``OUSAST_RUN`` and the Task's
|
||||
name (409 otherwise); one request per boot is accepted (202), later ones are 409, and a boot whose completion marker
|
||||
records a delivered run refuses every request (409 with the reason). Before the workspaces are ready the answer is
|
||||
503, which the reconciler retries. The credentials live in this process's memory and go into the command's
|
||||
environment only; they are never logged or written, and their values are redacted from the command's echoed stderr.
|
||||
``delivery`` (plane-on-kubernetes design section 3) is ``{"put": <presigned PUT URL for the output tar>, "inputs":
|
||||
{"<NAME>": <presigned GET URL>}}``: a presigned URL is a bearer capability, so it travels where the credentials do.
|
||||
Each URL must be ``https`` to a hostname (never an address; ``AX_RUNNER_INSECURE_DELIVERY=1`` admits ``http`` in unit
|
||||
tests only), the answer never echoes one, and no URL is logged.
|
||||
|
||||
The command runs as a child process, ``python -m openultrasast.plane.tasks.<spec.command[0]> <spec.command[1:]>``,
|
||||
in its own process group, with the first workspace as its working directory and ``AX_METADATA_URL`` pointing at
|
||||
this server. ax has no artifact channel and never reports that a command finished, so after the child exits the
|
||||
runner reads ``summary.json`` (a crash without one becomes ``failed`` with the exit code and the stderr tail),
|
||||
posts the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``OUSAST_ARTIFACT_URL`` -- that delivery is the task's
|
||||
PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` (or POSTs it to ``OUSAST_ARTIFACT_URL``, the receiver
|
||||
path kept until task 2.6 removes it) -- that delivery is the task's
|
||||
completion (Requirement 4.4); a failed one is reported ``failed``, never done -- and writes the completion marker
|
||||
``<OUSAST_STATE_DIR>/<task>.done.json`` (``exit``, ``status``, ``delivered``). Then it keeps serving until SIGTERM,
|
||||
as ax requires of PID 1. A boot that finds the marker never reruns the command (a resumed actor must not repeat
|
||||
@@ -38,6 +43,7 @@ Environment the runner reads, all optional except the two ax variables:
|
||||
AX_RUNNER_AUTOSTART "1" runs the command without waiting for a start request (unit tests only;
|
||||
required with AX_RUNNER_HTTP=0, where no request could arrive)
|
||||
AX_RUNNER_TASK_PACKAGE package the command's module is looked up in (unit tests only)
|
||||
AX_RUNNER_INSECURE_DELIVERY "1" admits http delivery URLs to a hostname (unit tests only; the image never sets it)
|
||||
OUSAST_STATE_DIR completion markers, default ``/workspace/.ousast-state`` (the durable volume)
|
||||
OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10
|
||||
OUSAST_CASE_CACHE a directory of git checkouts (``benchmarks/independent`` cache layout)
|
||||
@@ -47,8 +53,9 @@ Environment the runner reads, all optional except the two ax variables:
|
||||
``Host`` (the egress gateway decides on and resolves that name)
|
||||
OUSAST_DELIVERY_BACKOFF seconds before the second delivery attempt, doubled per retry
|
||||
OUSAST_INPUTS JSON ``{"<NAME>": "<producer>/<artifact>"}``: after the start and before the
|
||||
command each is fetched from ``<OUSAST_ARTIFACT_URL>/inputs/<ref>`` (same
|
||||
dial, headers and retries as delivery) to ``OUSAST_INPUT_<NAME>``; a failed
|
||||
command each is fetched from ``delivery.inputs[NAME]`` (else from
|
||||
``<OUSAST_ARTIFACT_URL>/inputs/<ref>`` with the receiver's dial and headers)
|
||||
to ``OUSAST_INPUT_<NAME>``, with the same retries as delivery; a failed
|
||||
fetch delivers a failed summary naming the input, the command never runs
|
||||
|
||||
The runner exports ``OUSAST_WORKSPACE_DIR`` (the first bound workspace) when the Task's env leaves it unset,
|
||||
@@ -60,6 +67,7 @@ from __future__ import annotations
|
||||
import contextlib
|
||||
import http.client
|
||||
import io
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -76,7 +84,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
from collections import deque
|
||||
from collections.abc import Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -206,14 +214,58 @@ _CREDENTIAL_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
|
||||
_RESERVED_PREFIXES = ("AX_", "OUSAST_") # the runner's own contract; a credential must not redirect it
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeliveryTargets:
|
||||
"""Where this task PUTs its output tar and GETs each input (presigned URLs from the start request)."""
|
||||
|
||||
put: str | None = None
|
||||
inputs: Mapping[str, str] = field(default_factory=dict)
|
||||
|
||||
def __repr__(self) -> str: # a presigned URL is a credential: never print one
|
||||
return f"DeliveryTargets(put={'yes' if self.put else 'no'}, inputs={sorted(self.inputs)})"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Start:
|
||||
"""An accepted start request: the credentials go to the command's environment and nowhere else."""
|
||||
"""An accepted start request: the credentials go to the command's environment and nowhere else; the delivery
|
||||
targets go to the artifact PUT and the input GETs and nowhere else."""
|
||||
|
||||
credentials: Mapping[str, str]
|
||||
delivery: DeliveryTargets | None = None
|
||||
|
||||
def __repr__(self) -> str: # never print a value by accident
|
||||
return f"Start(credentials={sorted(self.credentials)})"
|
||||
return f"Start(credentials={sorted(self.credentials)}, delivery={self.delivery!r})"
|
||||
|
||||
|
||||
def _check_url(url: object, where: str, insecure: bool) -> str:
|
||||
"""``url`` is a string, ``https`` (``http`` only when ``insecure``) to a hostname that is not an address, without
|
||||
userinfo; the error names the field and never echoes the value."""
|
||||
if not isinstance(url, str):
|
||||
raise ValueError(f"{where} must be a URL string")
|
||||
parts = urlsplit(url)
|
||||
allowed = ("https", "http") if insecure else ("https",)
|
||||
host = parts.hostname or ""
|
||||
if parts.scheme not in allowed or not host or parts.username is not None or parts.password is not None:
|
||||
raise ValueError(f"{where} must be an https URL to a hostname")
|
||||
try:
|
||||
ipaddress.ip_address(host.strip("[]"))
|
||||
except ValueError:
|
||||
return url
|
||||
raise ValueError(f"{where} must name a host, not an address")
|
||||
|
||||
|
||||
def parse_delivery(raw: object, insecure: bool = False) -> DeliveryTargets | None:
|
||||
"""The start body's ``delivery`` field validated: None when absent; :class:`ValueError` names the bad field."""
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, dict) or not set(raw) <= {"put", "inputs"}:
|
||||
raise ValueError("delivery must be an object with put and/or inputs")
|
||||
put = _check_url(raw["put"], "delivery.put", insecure) if raw.get("put") is not None else None
|
||||
inputs_raw = raw.get("inputs") or {}
|
||||
if not isinstance(inputs_raw, dict) or not all(isinstance(k, str) and _CREDENTIAL_NAME.fullmatch(k) for k in inputs_raw):
|
||||
raise ValueError("delivery.inputs must map ^[A-Z][A-Z0-9_]*$ names to URLs")
|
||||
inputs = {name: _check_url(url, f"delivery.inputs.{name}", insecure) for name, url in inputs_raw.items()}
|
||||
return DeliveryTargets(put, inputs)
|
||||
|
||||
|
||||
class StartGate:
|
||||
@@ -225,7 +277,8 @@ class StartGate:
|
||||
logged, and no answer echoes it.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, insecure: bool = False) -> None:
|
||||
self._insecure = insecure # AX_RUNNER_INSECURE_DELIVERY=1: http delivery URLs (unit tests only)
|
||||
self._lock = threading.Lock()
|
||||
self._accepted = threading.Event()
|
||||
self._expected: tuple[str, str] | None = None
|
||||
@@ -267,10 +320,14 @@ class StartGate:
|
||||
for k, v in creds.items()
|
||||
):
|
||||
return 400, "credentials must map ^[A-Z][A-Z0-9_]*$ names (not AX_/OUSAST_) to strings"
|
||||
try:
|
||||
delivery = parse_delivery(request.get("delivery"), self._insecure)
|
||||
except ValueError as exc:
|
||||
return 400, str(exc)
|
||||
run, task = self._expected
|
||||
if request.get("run") != run or request.get("task") != task:
|
||||
return 409, f"this actor runs task {task!r} of run {run!r}"
|
||||
self._start = Start(dict(creds))
|
||||
self._start = Start(dict(creds), delivery)
|
||||
self._accepted.set()
|
||||
return 202, "started"
|
||||
|
||||
@@ -653,6 +710,12 @@ def _dialled(url: str, headers: Mapping[str, str], dial: str | None) -> tuple[st
|
||||
return parts._replace(netloc=dial).geturl(), out
|
||||
|
||||
|
||||
def safe_url(url: str) -> str:
|
||||
"""``scheme://host/path`` of a URL: what a log line or an error may carry (a presigned query is a credential)."""
|
||||
parts = urlsplit(url)
|
||||
return f"{parts.scheme}://{parts.netloc}{parts.path}"
|
||||
|
||||
|
||||
def deliver(
|
||||
url: str,
|
||||
payload: bytes,
|
||||
@@ -661,15 +724,17 @@ def deliver(
|
||||
attempts: int = DELIVERY_ATTEMPTS,
|
||||
backoff: float = 1.0,
|
||||
dial: str | None = None,
|
||||
method: str = "POST",
|
||||
) -> int:
|
||||
"""POST ``payload`` to ``url`` with ``headers``; retry with doubling backoff; return the 2xx status.
|
||||
"""Send ``payload`` to ``url`` (``POST`` to the receiver, ``PUT`` to a presigned store URL) with ``headers``; retry
|
||||
with doubling backoff; return the 2xx status. Log lines and errors carry the URL without its query.
|
||||
|
||||
``dial`` (``host:port``): connect there instead and send the URL's host as ``Host``, as the actor must -- it
|
||||
cannot resolve the receiver's cluster name, and the egress gateway decides on ``Host`` and resolves it itself.
|
||||
"""
|
||||
last, (target, headers) = "no attempt made", _dialled(url, headers, dial)
|
||||
for attempt in range(1, attempts + 1):
|
||||
request = urllib.request.Request(target, data=payload, method="POST", headers=headers)
|
||||
request = urllib.request.Request(target, data=payload, method=method, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the reconciler's URL
|
||||
status = int(response.status)
|
||||
@@ -680,10 +745,10 @@ def deliver(
|
||||
last = f"HTTP {exc.code}"
|
||||
except (urllib.error.URLError, http.client.HTTPException, OSError, ValueError) as exc:
|
||||
last = f"{type(exc).__name__}: {exc}"
|
||||
log.warning("artifact delivery attempt %d/%d to %s failed: %s", attempt, attempts, url, last)
|
||||
log.warning("artifact delivery attempt %d/%d to %s failed: %s", attempt, attempts, safe_url(url), last)
|
||||
if attempt < attempts:
|
||||
time.sleep(backoff * (2 ** (attempt - 1)))
|
||||
raise DeliveryError(f"{last} after {attempts} attempts to {url}")
|
||||
raise DeliveryError(f"{last} after {attempts} attempts to {safe_url(url)}")
|
||||
|
||||
|
||||
def fetch(
|
||||
@@ -711,14 +776,15 @@ def fetch(
|
||||
except (urllib.error.URLError, http.client.HTTPException, OSError, ValueError) as exc:
|
||||
last = f"{type(exc).__name__}: {exc}"
|
||||
partial.unlink(missing_ok=True)
|
||||
log.warning("input fetch attempt %d/%d from %s failed: %s", attempt, attempts, url, last)
|
||||
log.warning("input fetch attempt %d/%d from %s failed: %s", attempt, attempts, safe_url(url), last)
|
||||
if attempt < attempts:
|
||||
time.sleep(backoff * (2 ** (attempt - 1)))
|
||||
raise DeliveryError(f"{last} after {attempts} attempts to {url}")
|
||||
raise DeliveryError(f"{last} after {attempts} attempts to {safe_url(url)}")
|
||||
|
||||
|
||||
def fetch_inputs(environ: Mapping[str, str]) -> list[Path]:
|
||||
"""Fetch every ``OUSAST_INPUTS`` entry from the receiver to its ``OUSAST_INPUT_<NAME>`` path.
|
||||
def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> list[Path]:
|
||||
"""Fetch every ``OUSAST_INPUTS`` entry to its ``OUSAST_INPUT_<NAME>`` path: from ``delivery.inputs[NAME]`` (a
|
||||
presigned GET, no headers) when the start request carries one, else from the receiver.
|
||||
|
||||
:class:`RunnerError` names the input when one cannot be fetched: a task never runs with an input missing."""
|
||||
raw = environ.get("OUSAST_INPUTS")
|
||||
@@ -736,11 +802,15 @@ def fetch_inputs(environ: Mapping[str, str]) -> list[Path]:
|
||||
fetched: list[Path] = []
|
||||
for name, ref in sorted(inputs.items()):
|
||||
dest = environ.get(f"OUSAST_INPUT_{name}")
|
||||
if not url or not dest or not isinstance(ref, str):
|
||||
raise RunnerError(f"input {name} ({ref}) not fetched: OUSAST_ARTIFACT_URL or OUSAST_INPUT_{name} is unset")
|
||||
source = f"{url.rstrip('/')}/inputs/{urllib.parse.quote(ref)}"
|
||||
presigned = delivery.inputs.get(name) if delivery is not None else None
|
||||
if not dest or not isinstance(ref, str) or not (presigned or url):
|
||||
raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL, OUSAST_ARTIFACT_URL or OUSAST_INPUT_{name}")
|
||||
try:
|
||||
size = fetch(source, headers, Path(dest), backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
|
||||
if presigned:
|
||||
size = fetch(presigned, {}, Path(dest), backoff=backoff)
|
||||
else:
|
||||
source = f"{str(url).rstrip('/')}/inputs/{urllib.parse.quote(ref)}"
|
||||
size = fetch(source, headers, Path(dest), backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
|
||||
except DeliveryError as exc:
|
||||
raise RunnerError(f"input {name} ({ref}) not fetched: {exc}") from None
|
||||
log.info("input %s (%s): %d bytes to %s", name, ref, size, dest)
|
||||
@@ -748,8 +818,13 @@ def fetch_inputs(environ: Mapping[str, str]) -> list[Path]:
|
||||
return fetched
|
||||
|
||||
|
||||
def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str]) -> bool:
|
||||
"""POST the output directory; False when there is nowhere to deliver; :class:`DeliveryError` when it fails."""
|
||||
def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool:
|
||||
"""PUT the output tar to ``delivery.put`` when the start request carried one, else POST it to the receiver;
|
||||
False when there is nowhere to deliver; :class:`DeliveryError` when it fails."""
|
||||
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
||||
if delivery is not None and delivery.put:
|
||||
deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff, method="PUT")
|
||||
return True
|
||||
url = environ.get("OUSAST_ARTIFACT_URL")
|
||||
if not url:
|
||||
return False
|
||||
@@ -758,7 +833,6 @@ def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str]) ->
|
||||
run = environ.get("OUSAST_RUN")
|
||||
if run:
|
||||
headers["X-Ousast-Run"] = run
|
||||
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
||||
deliver(url, build_tar(output_dir), headers, backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
|
||||
return True
|
||||
|
||||
@@ -810,14 +884,16 @@ def _exit_of(marker: Mapping[str, Any]) -> int:
|
||||
return EXIT_BY_STATUS.get(str(marker.get("status")), EXIT_BY_STATUS["failed"])
|
||||
|
||||
|
||||
def _finish(session: _Session, env: Mapping[str, str], exit_code: int | None, summary: Mapping[str, Any]) -> dict[str, Any]:
|
||||
def _finish(
|
||||
session: _Session, env: Mapping[str, str], exit_code: int | None, summary: Mapping[str, Any], delivery: DeliveryTargets | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Deliver the output directory and write the completion marker; a failed delivery reports ``failed``.
|
||||
|
||||
The marker keeps the task's own summary so a later boot can put it back and retry the delivery alone.
|
||||
"""
|
||||
marker: dict[str, Any] = {"exit": exit_code, "status": str(summary.get("status")), "delivered": False, "summary": dict(summary)}
|
||||
try:
|
||||
marker["delivered"] = _deliver_output(session.task, session.output_dir, env)
|
||||
marker["delivered"] = _deliver_output(session.task, session.output_dir, env, delivery)
|
||||
except DeliveryError as exc:
|
||||
log.error("artifact delivery failed: %s", exc)
|
||||
marker.update(status="failed", reason=f"artifact delivery failed: {exc}")
|
||||
@@ -857,24 +933,24 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa
|
||||
stored = done.get("summary")
|
||||
kept: dict[str, Any] = stored if isinstance(stored, dict) else read_summary(session.output_dir) or {}
|
||||
write_summary(session.output_dir, kept)
|
||||
return _exit_of(_finish(session, env, done.get("exit"), kept))
|
||||
return _exit_of(_finish(session, env, done.get("exit"), kept, start.delivery)) # a re-minted URL, if any
|
||||
if stop.is_set():
|
||||
return 0
|
||||
try:
|
||||
_materialise_once(session, env)
|
||||
fetch_inputs(env) # after the start as well: the receiver serves a task its inputs only while it runs
|
||||
fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it (and a receiver serves a running task only)
|
||||
except RunnerError as exc:
|
||||
log.error("runner failed: %s", exc)
|
||||
failed = _failed(f"runner failed: {exc}", read_summary(session.output_dir))
|
||||
write_summary(session.output_dir, failed)
|
||||
return _exit_of(_finish(session, env, None, failed))
|
||||
return _exit_of(_finish(session, env, None, failed, start.delivery))
|
||||
log.info("workspaces and inputs ready; running task %s: %s", session.task.metadata.name, list(session.task.command))
|
||||
grace = float(env.get("OUSAST_TERM_GRACE") or TERM_GRACE)
|
||||
code, summary = run_command(session.task, session.output_dir, env, stop, grace, start.credentials)
|
||||
if code is None:
|
||||
log.info("task %s was stopped before its command finished; a resume runs it again", session.task.metadata.name)
|
||||
return 0
|
||||
return _exit_of(_finish(session, env, code, summary))
|
||||
return _exit_of(_finish(session, env, code, summary, start.delivery))
|
||||
except RunnerError as exc:
|
||||
# The failure is reported, never acted on unasked: a start request is answered with it (409 without a
|
||||
# session, else 202 and the failed summary is delivered), so the golden boot delivers nothing either.
|
||||
@@ -887,9 +963,10 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa
|
||||
write_summary(session.output_dir, summary)
|
||||
gate.arm(env.get("OUSAST_RUN", ""), session.task.metadata.name)
|
||||
ready.set()
|
||||
if env.get("AX_RUNNER_AUTOSTART") != "1" and gate.wait(stop) is None:
|
||||
late = gate.autostart() if env.get("AX_RUNNER_AUTOSTART") == "1" else gate.wait(stop)
|
||||
if late is None:
|
||||
return 0
|
||||
return _exit_of(_finish(session, env, None, summary))
|
||||
return _exit_of(_finish(session, env, None, summary, late.delivery))
|
||||
|
||||
|
||||
def _materialise_once(session: _Session, env: Mapping[str, str]) -> None:
|
||||
@@ -918,14 +995,14 @@ def main(argv: Sequence[str] | None = None, environ: MutableMapping[str, str] |
|
||||
env = os.environ if environ is None else environ
|
||||
if argv:
|
||||
log.warning("ax-task-runner ignores command-line arguments %r; the Task manifest is the only input", list(argv))
|
||||
ready, stop, gate = threading.Event(), threading.Event(), StartGate()
|
||||
ready, stop, gate = threading.Event(), threading.Event(), StartGate(insecure=env.get("AX_RUNNER_INSECURE_DELIVERY") == "1")
|
||||
previous = _on_sigterm(stop)
|
||||
server: HealthServer | None = None
|
||||
if env.get("AX_RUNNER_HTTP", "1") != "0":
|
||||
metadata = {"task": env.get("AX_TASK_YAML", ""), "workspaces": env.get("AX_WORKSPACES_YAML", "")}
|
||||
server = start_health_server(int(env.get("AX_RUNNER_PORT", str(DEFAULT_PORT))), ready, metadata, gate)
|
||||
env["AX_RUNNER_BOUND_PORT"] = str(server.port)
|
||||
env["AX_METADATA_URL"] = f"http://127.0.0.1:{server.port}"
|
||||
env["AX_METADATA_URL"] = f"http://127.0.0.1:{server.port}" # loopback: this runner's own health server
|
||||
log.info("health and metadata server on port %d", server.port)
|
||||
try:
|
||||
if server is None and env.get("AX_RUNNER_AUTOSTART") != "1":
|
||||
|
||||
Reference in New Issue
Block a user