merge(plane-on-kubernetes): groups 1-2 -- cluster profile, no host literals, delivery through the S3 store

Kind proofs: records 01 and 02. Task 2.6 (receiver removal) is paused while the
safety net and the decision engine come first. deployment.md conflicts resolved
in the plain-language style; corrects the RuntimeClass claim (Substrate runs
runsc inside its worker pod).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-03 07:57:13 +02:00
co-authored by Claude Opus 5.5
62 changed files with 3080 additions and 433 deletions
+29 -10
View File
@@ -323,16 +323,21 @@ def _main(argv: list[str] | None) -> int:
learn_labels.add_argument("--include-title", action="store_true", help="the title-tier pairs as well (a sensitivity arm)")
learn_labels.add_argument("--no-assumed-benign", action="store_true", help="skip mining the assumed-benign history source")
add_learn_commands(learn_sub)
plane = subparsers.add_parser("plane", help="maintainer: the ax-backed service plane on this host's kind cluster")
plane = subparsers.add_parser("plane", help="maintainer: the ax-backed service plane on the profile's cluster (ops/k8s/profiles/)")
profile_help = "the PlaneProfile: a name under ops/k8s/profiles/ or a .toml path (default OUSAST_PLANE_PROFILE, else kind)"
plane_sub = plane.add_subparsers(dest="plane_command", required=True)
plane_run = plane_sub.add_parser("run", help="execute a Run manifest on ax; a rerun skips done tasks")
plane_run.add_argument("run_manifest", type=Path)
plane_run.add_argument("--workers", type=int, default=1, help="tasks in flight at once (serialize labels never overlap)")
plane_run.add_argument("--ax", default="ax", help="path of the ax CLI")
plane_run.add_argument("--profile", help=profile_help)
plane_status = plane_sub.add_parser("status", help="per-task status and the token attribution table of a run")
plane_status.add_argument("run")
plane_status.add_argument("--units", action="store_true", help="per-unit rows from each task's units.jsonl")
plane_sub.add_parser("doctor", help="check kind, Agent Substrate, the ax controller and the runner image")
plane_doctor = plane_sub.add_parser(
"doctor", help="check the profile's cluster, Agent Substrate, the ax controller, the runner image and the memory store"
)
plane_doctor.add_argument("--profile", help=profile_help)
plane_remember = plane_sub.add_parser("remember", help="ingest a run's memory rows into the store (OUSAST_MEMORY)")
plane_remember.add_argument("run")
plane_remember.add_argument("--plane", type=Path, default=Path("plane"), help="where runs/<run>.yaml names population and split")
@@ -370,9 +375,11 @@ def _main(argv: list[str] | None) -> int:
)
plane_harvest.add_argument("--ceiling", type=float, default=10.0, help="USD: each Run's task budgets sum to at most this")
plane_harvest.add_argument("--runner-image", type=Path, help="re-pin the task templates to the image in this file first (ops/ax/up.sh)")
plane_harvest.add_argument("--profile", help=profile_help)
plane_ws = plane_sub.add_parser("workspaces", help="one Workspace manifest per case pin of a population file")
plane_ws.add_argument("population", type=Path)
plane_ws.add_argument("--out", type=Path, default=Path("plane/workspaces"))
plane_ws.add_argument("--profile", help=profile_help)
plane_ws.add_argument("--validation-set", type=Path, help="JSON {case: [[path, function], ...]}: emit that set's increment instead")
plane_ws.add_argument("--candidates", type=Path, help="with --validation-set: per-case candidate records (independent-v2-modelsinks)")
plane_ws.add_argument("--triage", type=Path, help="with --validation-set: recorded batched-check scans (<case>--vulnerable_a.jsonl)")
@@ -1529,10 +1536,22 @@ def _learn_labels(args: argparse.Namespace) -> int:
def _plane(args: argparse.Namespace) -> int:
from .plane import reconciler # lazy: the plane is a maintainer surface, not the scan path
from .plane.profile import ProfileError, load_profile
try:
profile = load_profile(getattr(args, "profile", None))
except ProfileError as exc:
print(f"plane: {exc}", file=sys.stderr)
return 2
if args.plane_command == "run":
from .plane.memory import MemoryStoreError
_plane_memory("seed", args.run_manifest)
result = reconciler.run(args.run_manifest, workers=args.workers, ax=args.ax)
try:
result = reconciler.run(args.run_manifest, workers=args.workers, ax=args.ax, profile=profile)
except MemoryStoreError as exc: # the store is the state of record: no fallback, the Run is not submitted
print(f"memory store {profile.memory}: {exc}", file=sys.stderr)
return 1
print(f"run finished: {result}")
_plane_memory("ingest", args.run_manifest)
return 0 if result == "done" else 1
@@ -1547,14 +1566,14 @@ def _plane(args: argparse.Namespace) -> int:
if args.plane_command == "alerts-engine":
return _plane_alerts_engine(args)
if args.plane_command == "harvest":
return _plane_harvest(args)
return _plane_harvest(args, profile.atespace)
if args.plane_command == "doctor":
checks = reconciler.doctor()
checks = reconciler.doctor(profile)
for name, ok, text in checks:
print(f"[{'ok' if ok else 'FAIL'}] {name}: {text}")
return 0 if all(ok for _, ok, _ in checks) else 1
if args.validation_set is not None:
return _plane_increment(args)
return _plane_increment(args, profile.atespace)
from .plane.workspaces import workspaces
written = workspaces(args.population, args.out)
@@ -1562,7 +1581,7 @@ def _plane(args: argparse.Namespace) -> int:
return 0
def _plane_harvest(args: argparse.Namespace) -> int:
def _plane_harvest(args: argparse.Namespace, atespace: str) -> int:
from .plane.generate import repin_templates
from .plane.harvest import write_harvest
@@ -1571,7 +1590,7 @@ def _plane_harvest(args: argparse.Namespace) -> int:
command = f"ousast plane harvest --labels {args.labels.name} --name {args.name} --ceiling {args.ceiling:g}"
files = write_harvest(
args.labels, args.plane, name=args.name, command=command, catalog=args.catalog, cache=args.cache, templates=args.templates,
ceiling=args.ceiling,
ceiling=args.ceiling, atespace=atespace,
) # fmt: skip
for relative in sorted(k for k in files if k.startswith("runs/")):
print(f"{args.plane / relative}: {files[relative].splitlines()[1].lstrip('# ')}")
@@ -1652,7 +1671,7 @@ def _plane_memory(
return 0
def _plane_increment(args: argparse.Namespace) -> int:
def _plane_increment(args: argparse.Namespace, atespace: str) -> int:
from .plane.generate import LOOP_MANIFEST, Loop, increment
if args.candidates is None or args.triage is None:
@@ -1678,7 +1697,7 @@ def _plane_increment(args: argparse.Namespace) -> int:
command += "".join(f" --qualify-population {p}" for p in loop.populations)
written = increment(
args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command,
repos=args.repos, runner_image=args.runner_image, loop=loop,
repos=args.repos, runner_image=args.runner_image, loop=loop, atespace=atespace,
) # fmt: skip
print(f"{len(written)} manifests written under {args.plane} (Run {args.run})")
return 0
+3 -1
View File
@@ -162,7 +162,9 @@ class EngineSession:
return None
payload = json.dumps({"query": code}).encode()
call = urllib.request.Request(
f"http://127.0.0.1:{self._port}/query-sync", data=payload, headers={"Content-Type": "application/json"}
f"http://127.0.0.1:{self._port}/query-sync", # loopback: the Joern server this process started
data=payload,
headers={"Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(call, timeout=timeout) as response: # noqa: S310 -- fixed loopback URL
+192
View File
@@ -0,0 +1,192 @@
"""Task artifacts through the memory store (plane-on-kubernetes design section 3, Req 3.1).
A task's runner PUTs its output tar to a presigned URL the reconciler minted, and GETs each declared input from
another; the URLs travel in the start request's ``delivery`` field (a presigned URL is a bearer capability, so it
goes where the credentials go, never into an ActorTemplate's env). The reconciler polls the store for the tar,
extracts it under the run directory with the member checks the old receiver had, re-puts the task's declared
``outputs[]`` as single objects so a consumer's URL names exactly one artifact, and writes ``done.json``. The run's
``state.json`` is mirrored to the store after each change and seeded from it on a rerun elsewhere.
Store layout under the 30-day ``runs/`` expiry rule (``memory.RUNS_RULE_ID``)::
runs/<run>/state.json the reconciler's state, mirrored
runs/<run>/<task>/output.tar the runner's PUT
runs/<run>/<task>/<artifact> each declared output, re-put by the reconciler
runs/<run>/<task>/done.json status, bytes, file count, finished
runs/<run>/inputs/<name> an input no task produces (``put_input``)
Nothing here logs or prints a URL.
"""
from __future__ import annotations
import io
import json
import os
import re
import tarfile
from collections.abc import Iterable, Mapping
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from .manifests import Run
from .memory import RUNS_PREFIX, MemoryStore
__all__ = ["DONE", "GRACE_SECONDS", "OUTPUT_TAR", "STATE", "Delivery", "DeliveryError", "declared", "expiry", "input_var"]
OUTPUT_TAR = "output.tar"
DONE = "done.json"
STATE = "state.json"
INPUTS_DIR = "inputs"
GRACE_SECONDS = 600 # a URL outlives the task timeout by this much; a re-start after a re-resume re-mints anyway
_NAME = re.compile(r"[^A-Z0-9]+")
class DeliveryError(RuntimeError):
"""A delivered tar that cannot be accepted, or an object that is not there; the message never carries a URL."""
def expiry(environ: Mapping[str, str] | None = None) -> timedelta:
"""``OUSAST_TASK_TIMEOUT`` (7200 s) plus :data:`GRACE_SECONDS`."""
env = os.environ if environ is None else environ
return timedelta(seconds=float(env.get("OUSAST_TASK_TIMEOUT") or 7200) + GRACE_SECONDS)
def input_var(name: str) -> str:
"""The ``OUSAST_INPUT_<NAME>`` suffix of a Run input name, as the reconciler renders it."""
return _NAME.sub("_", name.upper())
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
def declared(run: Run) -> dict[str, list[str]]:
"""Per Run task, the artifacts to publish as single objects: its ``outputs[]`` plus every ``<producer>/<artifact>``
a consumer declares as an input on it (``summary.json`` of a producer is a usual one), in declaration order."""
wanted: dict[str, list[str]] = {entry.name: list(entry.outputs) for entry in run.tasks}
for entry in run.tasks:
for ref in entry.inputs.values():
producer, _, artifact = ref.partition("/")
if producer in wanted and artifact and artifact not in wanted[producer]:
wanted[producer].append(artifact)
return wanted
class Delivery:
"""Presigned URLs, polling, collection and state mirroring for one Run on one store."""
def __init__(self, store: MemoryStore, run: str, base: Path, *, expires: timedelta | None = None) -> None:
self.store, self.run, self.base = store, run, base
self.expires = expires or expiry()
def key(self, *parts: str) -> str:
return f"{RUNS_PREFIX}{self.run}/" + "/".join(parts)
@property
def host(self) -> str | None:
"""The hostname the runner will dial (for the task's egress policy); None for a store without one."""
return urlsplit(self.store.presign_get(self.key(STATE), timedelta(seconds=60))).hostname
# --- the start body -------------------------------------------------------------------------------------------
def put_url(self, task: str) -> str:
return self.store.presign_put(self.key(task, OUTPUT_TAR), self.expires)
def input_urls(self, inputs: Mapping[str, str]) -> dict[str, str]:
"""``{OUSAST_INPUT_<NAME> suffix: presigned GET of runs/<run>/<producer>/<artifact>}``."""
out: dict[str, str] = {}
for name, ref in inputs.items():
producer, _, artifact = ref.partition("/")
if not producer or not artifact:
raise DeliveryError(f"input {name}: {ref!r} is not <producer>/<artifact>")
out[input_var(name)] = self.store.presign_get(self.key(producer, artifact), self.expires)
return out
def body(self, task: str, inputs: Mapping[str, str]) -> dict[str, Any]:
"""The start request's ``delivery`` field for ``task``."""
return {"put": self.put_url(task), "inputs": self.input_urls(inputs)}
# --- completion ----------------------------------------------------------------------------------------------
def reset(self, task: str) -> None:
"""Before an attempt: remove the previous attempt's tar and ``done.json`` so a rerun (an unfinished task
resumed with a larger budget, an interrupted attempt) never completes on a stale object."""
for name in (OUTPUT_TAR, DONE):
self.store._delete(self.key(task, name))
def delivered(self, task: str) -> bool:
"""Whether the runner's tar is in the store (an exact-key listing, the store's HEAD)."""
key = self.key(task, OUTPUT_TAR)
return key in self.store._keys(key)
def collect(self, task: str, outputs: Iterable[str]) -> tuple[dict[str, Any] | None, int]:
"""Download and extract the tar under ``<base>/<task>/``, re-put the declared ``outputs``, write
``done.json``; returns (the extracted ``summary.json`` or None, the tar's byte size)."""
got = self.store._get(self.key(task, OUTPUT_TAR))
if got is None:
raise DeliveryError(f"task {task}: output.tar is not in the store")
data = got[0]
dest = (self.base / task).resolve()
try:
with tarfile.open(fileobj=io.BytesIO(data), mode="r:") as tar:
members = tar.getmembers()
for m in members:
if m.issym() or m.islnk() or not (dest / m.name).resolve().is_relative_to(dest):
raise DeliveryError(f"task {task}: rejected member {m.name}")
dest.mkdir(parents=True, exist_ok=True)
tar.extractall(dest, members) # every member checked above
except tarfile.TarError as exc:
raise DeliveryError(f"task {task}: bad tar: {exc}") from None
published = self.publish(task, outputs)
summary = _read_json(dest / "summary.json")
done = {
"task": task,
"status": summary.get("status") if isinstance(summary, dict) else None,
"bytes": len(data),
"files": sum(1 for m in members if m.isfile()),
"published": published,
"finished": _now(),
}
self.store._put(self.key(task, DONE), json.dumps(done, sort_keys=True).encode("utf-8"))
return (summary if isinstance(summary, dict) else None), len(data)
def publish(self, task: str, outputs: Iterable[str]) -> list[str]:
"""Re-put the task's declared outputs that exist under ``<base>/<task>/`` as single objects; the names put."""
put: list[str] = []
root = (self.base / task).resolve()
for artifact in outputs:
path = (root / artifact).resolve()
if path.is_relative_to(root) and path.is_file():
self.store._put(self.key(task, artifact), path.read_bytes())
put.append(artifact)
return put
def put_input(self, name: str, path: Path) -> str:
"""Store a file no task produces as ``runs/<run>/inputs/<name>``; returns its presigned GET URL."""
self.store._put(self.key(INPUTS_DIR, name), Path(path).read_bytes())
return self.store.presign_get(self.key(INPUTS_DIR, name), self.expires)
# --- state ---------------------------------------------------------------------------------------------------
def save_state(self, data: Mapping[str, Any]) -> None:
self.store._put(self.key(STATE), (json.dumps(data, indent=2) + "\n").encode("utf-8"))
def load_state(self) -> dict[str, Any] | None:
got = self.store._get(self.key(STATE))
if got is None:
return None
try:
loaded = json.loads(got[0].decode("utf-8"))
except ValueError:
return None
return loaded if isinstance(loaded, dict) else None
def _read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
+83 -13
View File
@@ -1,15 +1,35 @@
"""``ousast plane doctor``: four checks of this host's ax before a Run is submitted (ai-service-plane Req 3.5).
"""``ousast plane doctor``: checks of the configured cluster before a Run is submitted (ai-service-plane Req 3.5,
plane-on-kubernetes Req 1.2).
Everything is read from the :class:`~openultrasast.plane.profile.PlaneProfile`: the kube context is checked for
reachability and, when reachable, the two namespaces for ready pods; the ``images`` file's runner reference is
resolved in its registry (a local registry over its v2 API, a remote one through ``docker manifest inspect``); the
memory store is opened, which verifies an S3 bucket; and the first line reports every configured address. Nothing
here names a cluster, a registry or an address of its own: those are the profile's.
A sibling of ``reconciler.py`` (tasks.md task 0 allows either), which re-exports :func:`doctor`.
"""
from __future__ import annotations
import os
import ipaddress
import re
import subprocess
import urllib.error
import urllib.request
from .profile import PlaneProfile, ProfileError, load_profile
UP = "ops/ax/up.sh brings it up"
RUNBOOK = "docs/deployment.md lists the prerequisites"
_MANIFEST_TYPES = ", ".join(
(
"application/vnd.oci.image.index.v1+json",
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v2+json",
)
)
# --- checks ----------------------------------------------------------------------------------------------
@@ -34,18 +54,68 @@ def _pods_ready(namespace: str, context: str) -> tuple[bool, str]:
return not bad, f"not ready: {', '.join(bad)}" if bad else f"{len(rows)} pods ready"
def doctor() -> list[tuple[str, bool, str]]:
"""Four checks of the host's ax: kind, Agent Substrate, the ax controller, the runner image (Req 3.5)."""
context = "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
def _is_local(registry: str) -> bool:
host = registry.rsplit(":", 1)[0] if re.search(r":\d+$", registry) else registry
if host == "localhost":
return True
try:
return ipaddress.ip_address(host.strip("[]")).is_loopback or ipaddress.ip_address(host.strip("[]")).is_private
except ValueError:
return False
def _registry_has(image: str) -> tuple[bool, str]:
"""Resolve a digest-pinned reference ``<registry>/<repository>@sha256:<hex>``: a local registry answers its v2
API for the manifest by digest (plain HTTP, as kind's registry speaks); a remote one is asked through
``docker manifest inspect``, which carries the operator's registry login."""
name, _, digest = image.partition("@")
registry, _, repository = name.partition("/")
short = f"{repository}@{digest[:19]}..."
if _is_local(registry):
url = f"http://{registry}/v2/{repository}/manifests/{digest}"
request = urllib.request.Request(url, method="HEAD", headers={"Accept": _MANIFEST_TYPES})
try:
with urllib.request.urlopen(request, timeout=20) as response: # noqa: S310 - the profile's registry
found = 200 <= int(response.status) < 300
except urllib.error.HTTPError as exc:
return False, f"{short} missing from {registry} (HTTP {exc.code})"
except (urllib.error.URLError, OSError) as exc:
return False, f"{registry} unreachable ({str(exc)[:80]})"
return found, f"{short} present in {registry}" if found else f"{short} missing from {registry}"
ok, out = _sh("docker", "manifest", "inspect", image)
return ok, f"{short} present in {registry}" if ok else f"{short} not resolved in {registry} ({out[:120]})"
def doctor(profile: PlaneProfile | None = None) -> list[tuple[str, bool, str]]:
"""The checks of the profile's cluster: ``(name, ok, text)`` per line, the configured addresses first."""
profile = profile or load_profile()
hint = UP if profile.name == "kind" else RUNBOOK
context = profile.kube_context
checks = [(f"profile {profile.name}", True, " ".join(f"{k}={v}" for k, v in profile.addresses().items()))]
ok, out = _sh("kubectl", "--context", context, "cluster-info")
checks = [("kind cluster", ok, "reachable" if ok else f"context {context} unreachable ({out[:80]}); {UP}")]
for label, ns in (("agent substrate (ate-system)", "ate-system"), ("ax controller (ax-system)", "ax-system")):
ok, out = _pods_ready(ns, context)
checks.append((label, ok, out if ok else f"{out}; {UP}"))
ok, out = _sh("curl", "-s", "localhost:5001/v2/_catalog")
present = ok and "ousast-runner" in out
text = "ousast-runner present" if present else f"missing from localhost:5001 ({out[:80]}); {UP} and loads it"
checks.append(("runner image in kind registry", present, text))
checks.append((f"cluster {context}", ok, "reachable" if ok else f"context {context} unreachable ({out[:80]}); {hint}"))
if ok: # behind an unreachable context the namespaces cannot be told apart from it
for label, ns in (("agent substrate (ate-system)", "ate-system"), ("ax controller (ax-system)", "ax-system")):
ok, out = _pods_ready(ns, context)
checks.append((label, ok, out if ok else f"{out}; {hint}"))
try:
images = profile.load_images()
runner = images["runner"]
except ProfileError as exc:
checks.append((f"runner image in {profile.registry}", False, f"{exc}; {hint}"))
except KeyError:
checks.append((f"runner image in {profile.registry}", False, f"{profile.images} names no runner image; {hint}"))
else:
ok, out = _registry_has(runner)
checks.append((f"runner image in {profile.registry}", ok, out if ok else f"{out}; {hint} and loads it"))
from .memory import MemoryStoreError, open_store # lazy: boto3 only when the profile names an S3 store
try:
store = open_store(profile.memory)
except (MemoryStoreError, OSError, ValueError) as exc:
checks.append((f"memory store {profile.memory}", False, str(exc)[:300]))
else:
checks.append((f"memory store {profile.memory}", True, f"{store.describe()} opens (an S3 bucket is verified)"))
return checks
+41 -19
View File
@@ -84,17 +84,32 @@ def git_hosts(workspaces: Iterable[Workspace]) -> tuple[list[str], list[str]]:
return sorted(tls), sorted(plain)
def policy_for(task: Task, workspaces: Iterable[Workspace], model: Model | None, receiver_host: str | None) -> dict[str, Any]:
"""The task's EgressPolicy body: the receiver over HTTP/80, Git and Model hosts as TLS passthrough on 443."""
def policy_for(
task: Task,
workspaces: Iterable[Workspace],
model: Model | None,
store_host: str | None,
receiver_host: str | None = None,
) -> dict[str, Any]:
"""The task's EgressPolicy body (plane-on-kubernetes design section 3): one ``tlsPassthrough`` rule on 443 over
the sorted union of the store's hostname (``store_host``: where the runner PUTs its tar and GETs its inputs), the
bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. No ``http`` rule, unless
``receiver_host`` names the old receiver (the path task 2.6 removes). An http Git repo cannot be reached under
this policy and is refused; a store given as an address is refused naming ``S3_ENDPOINT`` (the gateway allows
hostnames only)."""
tls, plain = git_hosts(workspaces)
tls = sorted(set(tls) | set(model.egress_hosts if model else ()))
if receiver_host and not _is_ip(receiver_host): # an address cannot be allowed; only the direct test mode has one
plain = sorted({*plain, receiver_host.lower()})
rules: list[dict[str, Any]] = []
if plain:
rules.append({"http": {"hostnames": plain, "ports": {"numbers": [80]}}})
if tls:
rules.append({"tlsPassthrough": {"hostnames": tls, "ports": {"numbers": [443]}}})
raise ValueError(f"Task/{task.metadata.name}: http Git repos ({', '.join(plain)}) cannot be reached: the policy grants TLS only")
hosts = set(tls) | set(model.egress_hosts if model else ())
if store_host:
if _is_ip(store_host):
raise ValueError(f"Task/{task.metadata.name}: the store endpoint is an address ({store_host}); S3_ENDPOINT must be a hostname")
hosts.add(store_host.lower())
rules: list[dict[str, Any]] = []
if receiver_host and not _is_ip(receiver_host): # the receiver path, kept for group 2 only (task 2.6 deletes it)
rules.append({"http": {"hostnames": [receiver_host.lower()], "ports": {"numbers": [80]}}})
if hosts:
rules.append({"tlsPassthrough": {"hostnames": sorted(hosts), "ports": {"numbers": [443]}}})
return {"rules": rules}
@@ -110,9 +125,11 @@ def rules_of(policy: Mapping[str, Any]) -> list[dict[str, Any]]:
return sorted(out, key=lambda r: json.dumps(r, sort_keys=True))
def receiver_cluster_ip(context: str | None = None) -> str | None:
"""The ClusterIP of the ``ousast-receiver`` Service (``kubectl get svc``); None when kubectl cannot say."""
context = context or "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
def receiver_cluster_ip(context: str) -> str | None:
"""The ClusterIP of the ``ousast-receiver`` Service (``kubectl get svc`` in the profile's ``context``); None when
kubectl cannot say."""
if not context:
raise RuntimeError("receiver_cluster_ip needs the profile's kube context")
argv = ["kubectl", "--context", context, "-n", "ax-system", "get", "svc", "ousast-receiver", "-o", "jsonpath={.spec.clusterIP}"]
try:
proc = subprocess.run(argv, capture_output=True, text=True, check=False, timeout=20)
@@ -122,8 +139,9 @@ def receiver_cluster_ip(context: str | None = None) -> str | None:
return address if proc.returncode == 0 and _is_ip(address) else None
def receiver_address(port: int) -> tuple[str, str | None]:
"""(``OUSAST_ARTIFACT_URL``, ``OUSAST_ARTIFACT_DIAL``) for a receiver bound on ``port``.
def receiver_address(port: int, context: str | None) -> tuple[str, str | None]:
"""(``OUSAST_ARTIFACT_URL``, ``OUSAST_ARTIFACT_DIAL``) for a receiver bound on ``port``; ``context`` is the
profile's kube context the Service's ClusterIP is looked up in.
``OUSAST_ARTIFACT_HOST`` set: the old direct URL ``http://<host>:<port>/`` and no dial (tests, a host the actor
reaches without the gateway). Otherwise the receiver Service's name, dialled at ``OUSAST_ARTIFACT_DIAL`` or the
@@ -135,7 +153,9 @@ def receiver_address(port: int) -> tuple[str, str | None]:
return f"http://{host}:{port}/", None
dial = os.environ.get("OUSAST_ARTIFACT_DIAL")
if not dial:
address = receiver_cluster_ip()
if not context:
raise RuntimeError("receiver_address needs the profile's kube context to look the receiver Service up")
address = receiver_cluster_ip(context)
dial = f"{address}:80" if address else None
return f"http://{RECEIVER_HOST}/", dial
@@ -143,9 +163,11 @@ def receiver_address(port: int) -> tuple[str, str | None]:
class Egress:
"""``kubectl ate`` for one actor's egress policy; the executable is injectable so tests use a fake."""
def __init__(self, executable: str | None = None, context: str | None = None) -> None:
def __init__(self, executable: str | None, context: str) -> None:
if not context:
raise ValueError("Egress needs the profile's kube context; it never builds one")
self.executable = executable or os.environ.get("OUSAST_KUBECTL_ATE") or "kubectl-ate"
self.context = context or "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
self.context = context
def _run(self, *args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]:
argv = [self.executable, "--context", self.context, *args]
@@ -214,12 +236,12 @@ class Receiver(ThreadingHTTPServer):
daemon_threads = True
def __init__(self, run_name: str, base: Path, port: int) -> None:
def __init__(self, run_name: str, base: Path, port: int, context: str | None = None) -> None:
super().__init__(("0.0.0.0", port), _Handler)
self.run_name, self.base, self.running, self.delivered = run_name, base, set[str](), set[str]()
self.inputs: dict[str, frozenset[str]] = {}
self.lock = threading.Lock()
self.url, self.dial = receiver_address(self.server_address[1])
self.url, self.dial = receiver_address(self.server_address[1], context)
def _task(self, headers: Any) -> str | None:
"""The ``X-Ousast-Task`` of a request naming this run and a task that is running now, else None."""
+39 -20
View File
@@ -45,8 +45,9 @@ The recorded triage cost is derived per file: a batched-check record's ``usd`` p
hunts, and its ``usage`` counts only the hunts (``verify_sinks._Recording`` slices the client's usage from the
first hunt on), so the triage share is ``usd`` minus that usage priced at the reference model's list price.
``--runner-image FILE`` (``~/.cache/ousast/ax-src/runner-image``, written by ``ops/ax/up.sh``) re-pins the
templates' ``image:`` to the digest in that file before generating, so a rebuilt image is one command.
``--runner-image FILE`` (the profile's images file, ``ops/k8s/profiles/kind-images.json``, written by ``ops/ax/up.sh``;
the older one-line pin file still reads) re-pins the templates' ``image:`` to the runner digest in that file before
generating, so a rebuilt image is one command.
"""
from __future__ import annotations
@@ -82,7 +83,7 @@ TRIAGE_BASIS = (
RANGES_BASIS = "git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')"
HEADROOM = 1.5
CALLS_PER_HUNT = MAX_STEPS + 2 # every tool step, the answer, one empty-content retry
ATESPACE = "default"
ATESPACE = "default" # the default when no profile is given; ``increment`` takes ``profile.atespace``
CASE_PATH = "/workspace/case"
INPUTS_PATH = "/workspace/inputs"
FIXED_PATH = "/workspace/fixed"
@@ -236,10 +237,10 @@ def _case_record(item: CaseInputs) -> dict[str, Any]:
return record
def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
def _workspaces(item: CaseInputs, atespace: str = ATESPACE) -> tuple[dict[str, Any], dict[str, Any]]:
case = item.case
git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}]
pinned = _doc("Workspace", _ax_name(case["id"], "vulnerable"), {"git": git}, ATESPACE)
pinned = _doc("Workspace", _ax_name(case["id"], "vulnerable"), {"git": git}, atespace)
pinned["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={case['vulnerable']}"}
files = {
"candidates.json": _json({"id": case["id"], "family": case["family"], "candidates": [list(c) for c in item.kept]}),
@@ -249,7 +250,7 @@ def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
{"candidates": [list(c) for c in item.candidates], "kept": [list(c) for c in item.kept], "triage": dict(item.triage)}
),
}
inputs = _doc("Workspace", _ax_name(case["id"], "inputs"), {"files": [{"path": k, "content": v} for k, v in files.items()]}, ATESPACE)
inputs = _doc("Workspace", _ax_name(case["id"], "inputs"), {"files": [{"path": k, "content": v} for k, v in files.items()]}, atespace)
return pinned, inputs
@@ -339,11 +340,11 @@ def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str,
return task, entry
def _alerts(item: CaseInputs, templates: Mapping[str, Task]) -> tuple[dict, dict, dict]:
def _alerts(item: CaseInputs, templates: Mapping[str, Task], atespace: str = ATESPACE) -> tuple[dict, dict, dict]:
"""(fixed-pin Workspace, Task, Run entry) of the case's `alerts`: quick-mode rules on both pins."""
case = item.case
git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}]
fixed = _doc("Workspace", _ax_name(case["id"], "fixed"), {"git": git}, ATESPACE)
fixed = _doc("Workspace", _ax_name(case["id"], "fixed"), {"git": git}, atespace)
fixed["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={case['fixed']}"}
bindings = [(_ax_name(case["id"], "vulnerable"), CASE_PATH), (fixed["metadata"]["name"], FIXED_PATH)]
bindings.append((_ax_name(case["id"], "inputs"), INPUTS_PATH))
@@ -358,10 +359,12 @@ def _alerts(item: CaseInputs, templates: Mapping[str, Task]) -> tuple[dict, dict
return fixed, task, entry
def _loop(cases: Sequence[CaseInputs], templates: Mapping[str, Task], loop: Loop) -> tuple[dict, list[dict], list[dict]]:
def _loop(
cases: Sequence[CaseInputs], templates: Mapping[str, Task], loop: Loop, atespace: str = ATESPACE
) -> tuple[dict, list[dict], list[dict]]:
"""(project Workspace, Tasks, Run entries) of the loop's singletons: snapshot -> measure -> propose -> improve."""
git = [{"name": "repo", "repo": loop.repo, "dir": "repo", "depth": 1}]
project = _doc("Workspace", _ax_name("openultrasast", loop.commit[:12]), {"git": git}, ATESPACE)
project = _doc("Workspace", _ax_name("openultrasast", loop.commit[:12]), {"git": git}, atespace)
project["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={loop.commit}"}
bound = [(project["metadata"]["name"], PROJECT_PATH)]
env = {
@@ -399,6 +402,7 @@ def render(
population: str = "",
split: str = "",
loop: Loop | None = None,
atespace: str = ATESPACE,
) -> dict[str, str]:
"""Relative path under the plane root -> file text, for every generated manifest. ``population`` and ``split``
(the population file's and the set's stems) become the Run's annotations, which the memory rows carry. Each
@@ -411,7 +415,7 @@ def render(
tiebreak: list[dict[str, Any]] = []
for item in cases:
case_id = item.case["id"]
pinned, inputs = _workspaces(item)
pinned, inputs = _workspaces(item, atespace)
for doc in (pinned, inputs):
out[f"workspaces/{doc['metadata']['name']}.yaml"] = _dump(header, [doc])
repo = [(pinned["metadata"]["name"], CASE_PATH), (inputs["metadata"]["name"], INPUTS_PATH)]
@@ -451,7 +455,7 @@ def render(
entries.extend(tiebreak) # appended: the first Run's entries stay byte-identical, so a rerun keeps their state
for item in cases: # appended after the tie-break for the same reason
if loop is not None:
fixed, alerts_task, alerts_entry = _alerts(item, templates)
fixed, alerts_task, alerts_entry = _alerts(item, templates, atespace)
out[f"workspaces/{fixed['metadata']['name']}.yaml"] = _dump(header, [fixed])
tasks.append(alerts_task)
entries.append(alerts_entry)
@@ -462,7 +466,7 @@ def render(
tasks.append(remember_task)
entries.append(remember_entry)
if loop is not None:
project, loop_tasks, loop_entries = _loop(cases, templates, loop)
project, loop_tasks, loop_entries = _loop(cases, templates, loop, atespace)
out[f"workspaces/{project['metadata']['name']}.yaml"] = _dump(header, [project])
tasks.extend(loop_tasks)
entries.extend(loop_entries)
@@ -482,11 +486,18 @@ _IMAGE_LINE = re.compile(r'^(\s*image:\s*)"?[^"\s]+"?\s*$', re.MULTILINE)
def repin_templates(plane: Path, image_file: Path) -> str:
"""Set every template's ``image:`` to the digest-pinned reference in ``image_file`` (as ``ops/ax/up.sh``
writes it); comments and layout are kept. Returns the reference."""
image = image_file.read_text(encoding="utf-8").strip()
if not re.fullmatch(r"\S+@sha256:[0-9a-f]{64}", image):
raise ValueError(f"{image_file}: expected a digest-pinned image reference, got {image!r}")
"""Set every template's ``image:`` to the runner reference of ``image_file``: a profile images file
(``{"runner": "<registry>/<name>@sha256:..."}``, as ``ops/ax/up.sh`` writes it) or the older one-line pin; a
reference without a digest is refused. Comments and layout are kept. Returns the reference. (Task 4.3 gives
the engine template ``images["engine"]``; until then only the runner key is read.)"""
from .profile import ProfileError, load_images
try:
image = load_images(image_file)["runner"]
except ProfileError as exc:
raise ValueError(str(exc)) from None
except KeyError:
raise ValueError(f"{image_file}: no runner image in the images file") from None
for name in TEMPLATES:
path = plane / "tasks" / f"{name}.yaml"
text = path.read_text(encoding="utf-8")
@@ -509,16 +520,24 @@ def increment(
repos: Path | None = None,
runner_image: Path | None = None,
loop: Loop | None = None,
atespace: str | None = None,
) -> list[Path]:
"""Write the generated manifests under ``plane`` from the templates in ``plane/tasks`` (re-pinned first when
``runner_image`` is given), with the improvement loop when ``loop`` is given; returns the paths."""
``runner_image`` is given), with the improvement loop when ``loop`` is given; returns the paths. Workspaces go
to ``atespace`` (default: the profile's)."""
if atespace is None:
from .profile import load_profile
atespace = load_profile().atespace
if loop is not None and repos is None:
raise ValueError("--loop needs --repos: the alerts task marks fixed-pin alerts by the fix's new-side ranges")
if runner_image is not None:
repin_templates(plane, runner_image)
loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES])
cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos, fixed_side=loop is not None)
files = render(cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem, loop=loop)
files = render(
cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem, loop=loop, atespace=atespace
)
written: list[Path] = []
for relative, text in files.items():
path = plane / relative
+26 -15
View File
@@ -210,17 +210,19 @@ def _reader(cache: Path, source: str, ref: str, pin: str) -> Any:
# --- manifests -------------------------------------------------------------------------------------------------------
def _git_workspace(unit: Unit) -> dict[str, Any]:
doc = _doc("Workspace", _ax_name(unit.name, "pin"), {"git": [{"name": REPO, "repo": unit.repo, "dir": REPO, "depth": 1}]}, ATESPACE)
def _git_workspace(unit: Unit, atespace: str = ATESPACE) -> dict[str, Any]:
doc = _doc("Workspace", _ax_name(unit.name, "pin"), {"git": [{"name": REPO, "repo": unit.repo, "dir": REPO, "depth": 1}]}, atespace)
doc["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"{REPO}={unit.pin}"}
return doc
def _files_workspace(name: str, files: Mapping[str, str]) -> dict[str, Any]:
return _doc("Workspace", name, {"files": [{"path": k, "content": v} for k, v in sorted(files.items())]}, ATESPACE)
def _files_workspace(name: str, files: Mapping[str, str], atespace: str = ATESPACE) -> dict[str, Any]:
return _doc("Workspace", name, {"files": [{"path": k, "content": v} for k, v in sorted(files.items())]}, atespace)
def _verify_docs(unit: Unit, templates: Mapping[str, Task], usd_per_hunt: float) -> tuple[list[dict], list[dict], list[dict]]:
def _verify_docs(
unit: Unit, templates: Mapping[str, Task], usd_per_hunt: float, atespace: str = ATESPACE
) -> tuple[list[dict], list[dict], list[dict]]:
"""(Workspaces, Tasks, Run entries) of one verify unit."""
asked = list(unit.verify or unit.candidates)
candidates = {"family": unit.family, "candidates": [list(c) for c in asked]}
@@ -232,15 +234,15 @@ def _verify_docs(unit: Unit, templates: Mapping[str, Task], usd_per_hunt: float)
entries: list[dict] = []
if unit.files:
files = {f"{REPO}/{path}": text for path, text in unit.files} | {"inputs/candidates.json": _json(candidates)}
workspace = _files_workspace(_ax_name(unit.name, "side"), files)
workspace = _files_workspace(_ax_name(unit.name, "side"), files, atespace)
workspaces.append(workspace)
bindings = [(workspace["metadata"]["name"], CASE_PATH)]
inputs = f"{CASE_PATH}/inputs/candidates.json"
else:
pinned = _git_workspace(unit)
pinned = _git_workspace(unit, atespace)
functions = [{"path": p, "function": fn, "line": ln} for p, fn, ln in asked]
files_doc = _files_workspace(
_ax_name(unit.name, "inputs"), {"candidates.json": _json(candidates), "functions.json": _json(functions)}
_ax_name(unit.name, "inputs"), {"candidates.json": _json(candidates), "functions.json": _json(functions)}, atespace
)
workspaces += [pinned, files_doc]
bindings = [(pinned["metadata"]["name"], CASE_PATH), (files_doc["metadata"]["name"], INPUTS_PATH)]
@@ -309,7 +311,7 @@ def _roles_groups(units: Sequence[Unit]) -> list[list[Unit]]:
def render_harvest(
units: Sequence[Unit], templates: Mapping[str, Task], name: str, command: str, *, ceiling: float = 10.0
units: Sequence[Unit], templates: Mapping[str, Task], name: str, command: str, *, ceiling: float = 10.0, atespace: str = ATESPACE
) -> dict[str, str]:
"""Relative path under the plane root -> file text: both Runs, their Workspaces and Tasks, and ``units.json``.
A population file whose size is unknown is estimated at :data:`UNKNOWN_SIZE`."""
@@ -323,7 +325,7 @@ def render_harvest(
tasks: list[dict] = []
entries: list[dict] = []
for unit in verify_units:
w, t, e = _verify_docs(unit, templates, USD_PER_HUNT[_kind(unit)] * scale)
w, t, e = _verify_docs(unit, templates, USD_PER_HUNT[_kind(unit)] * scale, atespace)
workspaces += w
tasks += t
entries += e
@@ -333,7 +335,7 @@ def render_harvest(
for group in _roles_groups([u for u in units if u.files and _inline(u, roles=True) <= MAX_INLINE_BYTES]):
files = {f"{REPO}/{u.name}/{p}": t for u in group for p, t in u.files if p in u.paths}
listing = sorted(f.removeprefix(f"{REPO}/") for f in files)
workspace = _files_workspace(_ax_name(name, "roles", group[0].name), files | {"inputs/files.json": _json(listing)})
workspace = _files_workspace(_ax_name(name, "roles", group[0].name), files | {"inputs/files.json": _json(listing)}, atespace)
workspaces.append(workspace)
estimates.append(_roles_estimate((len(text), len(text.splitlines())) for text in files.values()))
bound = [(workspace["metadata"]["name"], CASE_PATH)]
@@ -350,8 +352,8 @@ def render_harvest(
same = [u for u in units if not u.files and (u.repo, u.pin) == (unit.repo, unit.pin)]
paths = sorted({p for u in same for p in u.paths})
known = {p: (chars, lines) for u in same for p, chars, lines in u.sizes}
pinned = _git_workspace(unit)
files_doc = _files_workspace(_ax_name(unit.name, "roles-in"), {"files.json": _json(paths)})
pinned = _git_workspace(unit, atespace)
files_doc = _files_workspace(_ax_name(unit.name, "roles-in"), {"files.json": _json(paths)}, atespace)
workspaces += [w for w in (pinned, files_doc) if w["metadata"]["name"] not in emitted]
estimates.append(_roles_estimate(known.get(p, UNKNOWN_SIZE) for p in paths))
bound = [(pinned["metadata"]["name"], CASE_PATH), (files_doc["metadata"]["name"], INPUTS_PATH)]
@@ -380,7 +382,16 @@ def _run_files(header: str, run: str, workspaces: list[dict], tasks: list[dict],
def write_harvest(
labels: Path, plane: Path, *, name: str, command: str, catalog: Path, cache: Path, templates: Path, ceiling: float = 10.0
labels: Path,
plane: Path,
*,
name: str,
command: str,
catalog: Path,
cache: Path,
templates: Path,
ceiling: float = 10.0,
atespace: str = ATESPACE,
) -> dict[str, str]:
"""Build the units, render, write under ``plane`` (with the Model manifests copied beside); returns the rendered
files."""
@@ -388,7 +399,7 @@ def write_harvest(
units = pair_units(labels, catalog) + population_units(labels, cache)
loaded = load_manifests([templates / "tasks" / f"{n}.yaml" for n in ("repo-facts", "verify", "agree", "roles")]).tasks
files = render_harvest(units, loaded, name, command, ceiling=ceiling)
files = render_harvest(units, loaded, name, command, ceiling=ceiling, atespace=atespace)
for path in sorted((templates / "models").glob("*.yaml")):
files[f"models/{path.name}"] = path.read_text(encoding="utf-8")
for relative, text in files.items():
+266
View File
@@ -0,0 +1,266 @@
"""The plane's configuration surface: one ``PlaneProfile`` per cluster (plane-on-kubernetes design section 1).
A profile is a TOML file under ``ops/k8s/profiles/`` (``kind.toml`` for the laptop's kind cluster, ``k3s.toml`` for
the production cluster) holding every value that differs between clusters: the execution mode, the kube context,
the registry and the digest-pinned images file, the router and ax-server addresses (empty: tunnel through the
Kubernetes API), the memory store, the atespace, ax's snapshot bucket, the worker pools and the image pull secret.
:func:`load_profile` reads the file ``OUSAST_PLANE_PROFILE`` names (a profile name or a path; default ``kind``) and
then applies the environment: every scalar field is overridable by ``OUSAST_<FIELD>`` (``OUSAST_KUBE_CONTEXT``,
``OUSAST_REGISTRY``, ``OUSAST_MEMORY``, ``OUSAST_ROUTER_URL``, ...), a pool by ``OUSAST_POOL_<NAME>_<FIELD>``.
A profile holds no secret: a key whose name ends in ``key``, ``secret``, ``token`` or ``password`` is refused, and so
is any unknown key, an ``exec`` outside ``local``/``remote``, and an images file whose references are not pinned by
digest (Agent Substrate rejects tags). Provider and store credentials stay in the environment or ``.env``.
``python -m openultrasast.plane.profile --print FIELD`` prints one value for shell scripts (``ops/ax/up.sh``):
``registry``, ``kube_context``, ``images.runner``, ``pools.default.replicas``, ...
"""
from __future__ import annotations
import json
import os
import re
import sys
import tomllib
from collections.abc import Mapping
from dataclasses import dataclass, field, fields
from pathlib import Path
from typing import Any
__all__ = ["DEFAULT_PROFILE", "PROFILES_DIR", "PlaneProfile", "Pool", "ProfileError", "load_images", "load_profile", "profiles_dir"]
DEFAULT_PROFILE = "kind"
PROFILES_DIR = Path("ops/k8s/profiles")
EXEC_MODES = ("local", "remote")
_DIGEST = re.compile(r"\S+@sha256:[0-9a-f]{64}")
_SECRET_KEY = re.compile(r"(key|secret|token|password)$", re.IGNORECASE)
_SECRET_VALUE = re.compile(r"(sk-[A-Za-z0-9]{8,}|AKIA[0-9A-Z]{12,}|[A-Za-z0-9+/]{40,}={0,2}$)")
_ENV_PREFIX = "OUSAST_"
class ProfileError(ValueError):
"""A profile that cannot be used; the message names the file and the field."""
@dataclass(frozen=True)
class Pool:
"""One WorkerPool's size: how many warm workers, and each worker's CPU and memory limit."""
replicas: int
cpu: str
memory: str
@dataclass(frozen=True)
class PlaneProfile:
"""The values of one cluster. ``images`` is resolved against the profile file's directory."""
name: str
path: Path
exec: str = "local"
kube_context: str = ""
registry: str = ""
images: Path = Path("images.json")
router_url: str = ""
ax_server: str = ""
memory: str = ""
atespace: str = "default"
snapshots_bucket: str = ""
image_pull_secret: str = ""
pools: Mapping[str, Pool] = field(default_factory=dict)
def load_images(self) -> dict[str, str]:
"""The digest-pinned image references of ``images`` (``{"runner": ..., "engine": ...}``)."""
return load_images(self.images)
def addresses(self) -> dict[str, str]:
"""Every configured address, for ``ousast plane doctor`` (Req 1.2); an empty router or ax-server address
means the Kubernetes API tunnel."""
return {
"exec": self.exec,
"kube_context": self.kube_context,
"registry": self.registry,
"images": str(self.images),
"router_url": self.router_url or "(tunnel through the kube context)",
"ax_server": self.ax_server or "(tunnel through the kube context)",
"memory": self.memory,
"atespace": self.atespace,
"snapshots_bucket": self.snapshots_bucket,
"image_pull_secret": self.image_pull_secret or "(none)",
}
_SCALARS = {f.name: str(f.type) for f in fields(PlaneProfile) if f.name not in ("name", "path", "pools")}
_POOL_FIELDS = {f.name: str(f.type) for f in fields(Pool)}
def profiles_dir() -> Path:
"""``ops/k8s/profiles`` of this checkout (``OUSAST_PLANE_PROFILES`` overrides; an installed package falls back
to the working directory)."""
override = os.environ.get("OUSAST_PLANE_PROFILES")
if override:
return Path(override)
checkout = Path(__file__).resolve().parents[3] / PROFILES_DIR
return checkout if checkout.is_dir() else PROFILES_DIR
def load_images(path: Path) -> dict[str, str]:
"""``{"runner": "<registry>/<name>@sha256:<64 hex>", ...}`` from a JSON images file; every value must be
digest-pinned. A one-line file holding a single reference (``ops/ax/up.sh`` before this profile) reads as
``{"runner": <line>}``."""
try:
text = path.read_text(encoding="utf-8").strip()
except OSError as exc:
raise ProfileError(f"images file {path}: {exc.strerror or exc}") from None
try:
loaded: Any = json.loads(text) if text.startswith("{") else {"runner": text}
except ValueError as exc:
raise ProfileError(f"images file {path}: not JSON ({exc})") from None
if not isinstance(loaded, dict) or not loaded or not all(isinstance(k, str) and isinstance(v, str) for k, v in loaded.items()):
raise ProfileError(f"images file {path}: expected an object of image names to references")
for name, reference in loaded.items():
if not _DIGEST.fullmatch(reference):
raise ProfileError(f"images file {path}: {name} is not digest-pinned ({reference!r}); Agent Substrate rejects tags")
return dict(loaded)
def _reject_secret(where: str, key: str, value: object, declared: bool = False) -> None:
"""A key that is not a declared field and is named like a credential, or any value shaped like one, is refused.
``image_pull_secret`` is declared: its value is the *name* of a Kubernetes Secret, never its content."""
if not declared and _SECRET_KEY.search(key):
raise ProfileError(f"{where}: {key} looks like a credential; profiles hold no secret (use the environment or .env)")
if isinstance(value, str) and _SECRET_VALUE.search(value):
raise ProfileError(f"{where}: the value of {key} looks like a credential; profiles hold no secret")
def _coerce(where: str, key: str, value: object, kind: str) -> Any:
if kind == "int":
if isinstance(value, bool) or not isinstance(value, int):
raise ProfileError(f"{where}: {key} must be an integer, got {value!r}")
return value
if kind == "Path":
if not isinstance(value, str) or not value:
raise ProfileError(f"{where}: {key} must be a file name")
return Path(value)
if not isinstance(value, str):
raise ProfileError(f"{where}: {key} must be a string, got {value!r}")
return value
def _pools(where: str, raw: object) -> dict[str, Pool]:
if not isinstance(raw, dict):
raise ProfileError(f"{where}: pools must be a table of pool names")
out: dict[str, Pool] = {}
for name, spec in raw.items():
if not isinstance(spec, dict):
raise ProfileError(f"{where}: pools.{name} must be a table")
unknown = sorted(set(spec) - set(_POOL_FIELDS))
if unknown:
raise ProfileError(f"{where}: pools.{name} has unknown keys: {', '.join(unknown)}")
missing = sorted(set(_POOL_FIELDS) - set(spec))
if missing:
raise ProfileError(f"{where}: pools.{name} lacks {', '.join(missing)}")
for key, value in spec.items():
_reject_secret(where, f"pools.{name}.{key}", value, declared=True)
out[str(name)] = Pool(**{k: _coerce(where, f"pools.{name}.{k}", spec[k], _POOL_FIELDS[k]) for k in _POOL_FIELDS})
return out
def _env_overrides(environ: Mapping[str, str], where: str, values: dict[str, Any], pools: dict[str, Pool]) -> None:
for name, kind in _SCALARS.items():
raw = environ.get(_ENV_PREFIX + name.upper())
if raw is not None and raw != "":
value: Any = raw
if kind == "int":
try:
value = int(raw)
except ValueError:
raise ProfileError(f"{where}: {_ENV_PREFIX}{name.upper()} must be an integer, got {raw!r}") from None
values[name] = _coerce(where, _ENV_PREFIX + name.upper(), value, kind)
pattern = re.compile(rf"^{_ENV_PREFIX}POOL_([A-Z0-9]+)_(REPLICAS|CPU|MEMORY)$")
for variable, raw in environ.items():
found = pattern.match(variable)
if not found or raw == "":
continue
pool_name, key = found.group(1).lower(), found.group(2).lower()
current = pools.get(pool_name)
if current is None:
raise ProfileError(f"{where}: {variable} names a pool the profile does not declare ({pool_name})")
value = int(raw) if key == "replicas" and raw.isdigit() else raw
pools[pool_name] = Pool(**{**vars(current), key: _coerce(where, variable, value, _POOL_FIELDS[key])})
def load_profile(name: str | None = None, environ: Mapping[str, str] | None = None) -> PlaneProfile:
"""The profile ``name`` (a name under :func:`profiles_dir` or a path to a ``.toml``; default
``OUSAST_PLANE_PROFILE``, else ``kind``), with the environment's overrides applied; :class:`ProfileError`
names what is wrong."""
env = os.environ if environ is None else environ
chosen = name or env.get("OUSAST_PLANE_PROFILE") or DEFAULT_PROFILE
path = Path(chosen) if chosen.endswith(".toml") or "/" in chosen else profiles_dir() / f"{chosen}.toml"
where = str(path)
try:
raw = tomllib.loads(path.read_text(encoding="utf-8"))
except OSError as exc:
raise ProfileError(f"profile {where}: {exc.strerror or exc}") from None
except tomllib.TOMLDecodeError as exc:
raise ProfileError(f"profile {where}: {exc}") from None
for key, value in raw.items(): # a credential-shaped key is named as such before it is merely unknown
_reject_secret(where, key, value, declared=key in _SCALARS)
unknown = sorted(set(raw) - set(_SCALARS) - {"pools"})
if unknown:
raise ProfileError(f"{where}: unknown keys: {', '.join(unknown)}")
values: dict[str, Any] = {}
for key, value in raw.items():
if key != "pools":
values[key] = _coerce(where, key, value, _SCALARS[key])
pools = _pools(where, raw.get("pools", {}))
_env_overrides(env, where, values, pools)
if values.get("exec", "local") not in EXEC_MODES:
raise ProfileError(f"{where}: exec must be one of {', '.join(EXEC_MODES)}, got {values.get('exec')!r}")
images = Path(values.get("images", "images.json"))
if not images.is_absolute():
images = path.resolve().parent / images
values["images"] = images
if images.is_file():
load_images(images) # a committed file must be pinned; a file the release has not published yet is doctor's finding
return PlaneProfile(name=path.stem, path=path, pools=pools, **values)
def _lookup(profile: PlaneProfile, dotted: str) -> str:
head, _, rest = dotted.partition(".")
if head == "images" and rest:
images = profile.load_images()
if rest not in images:
raise ProfileError(f"{profile.images}: no image named {rest!r} (have {', '.join(sorted(images))})")
return images[rest]
if head == "pools":
pool_name, _, key = rest.partition(".")
if pool_name not in profile.pools or key not in _POOL_FIELDS:
raise ProfileError(f"{profile.path}: no such pool field {dotted!r}")
return str(getattr(profile.pools[pool_name], key))
if head not in _SCALARS or rest:
raise ProfileError(f"{profile.path}: no such field {dotted!r}")
return str(getattr(profile, head))
def main(argv: list[str] | None = None) -> int:
"""``python -m openultrasast.plane.profile [--profile NAME] --print FIELD [FIELD...]``: one value per line."""
import argparse
parser = argparse.ArgumentParser(prog="openultrasast.plane.profile", description=main.__doc__)
parser.add_argument("--profile", help="profile name or path (default OUSAST_PLANE_PROFILE, else kind)")
parser.add_argument("--print", dest="fields", nargs="+", required=True, metavar="FIELD")
args = parser.parse_args(argv)
try:
profile = load_profile(args.profile)
for dotted in args.fields:
print(_lookup(profile, dotted))
except ProfileError as exc:
print(f"profile: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
sys.exit(main())
+113 -41
View File
@@ -1,23 +1,26 @@
"""ax-backed reconciler of a ``Run``: submission, resume, start, completion by delivery (ai-service-plane Req 3, 4, 7).
ax executes every task on this host's kind cluster; nothing here runs a task locally or knows a prompt, a score or
a model call. Per ready task: one rendered YAML file (the Task with its env extended, its Workspaces, the bound
Model), ``ax apply -f``, ``ax resume task`` (retried on DeadlineExceeded/Unavailable -- a new image's first resume
waits for Agent Substrate's golden snapshot -- for up to ``OUSAST_RESUME_TIMEOUT`` s, 900), and once ax says
ax executes every task on the profile's cluster (``profile.py``: kube context, registry, store); nothing here runs a
task locally or knows a prompt, a score or a model call. Per ready task: one rendered YAML file (the Task with its
env extended, its Workspaces, the bound Model), ``ax apply -f``, ``ax resume task`` (retried on
DeadlineExceeded/Unavailable -- a new image's first resume waits for Agent Substrate's golden snapshot -- for up to
``OUSAST_RESUME_TIMEOUT`` s, 900), and once ax says
``Running`` one start request through Substrate's router (``router.py``, Req 4.5) carrying the bound Model's
credential from this process's environment, never from a file (Req 4.6). Completion is the runner's artifact
delivery (ax has no Completed phase); ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s
(7200) without delivery fail it; ``ax delete task`` follows.
credential from this process's environment, never from a file (Req 4.6), and the presigned store URLs of
``delivery.py`` (plane-on-kubernetes design section 3). Completion is the runner's output tar appearing in the
memory store (ax has no Completed phase), which the reconciler polls, downloads, extracts and re-publishes per
declared output; ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without
delivery fail it; ``ax delete task`` follows. ``OUSAST_DELIVERY=receiver`` keeps the host receiver of the kind
profile for group 2 only (task 2.6 removes it).
ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the
task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's pin annotation as
``OUSAST_GIT_PINS``. Inputs other tasks produced are never rendered: ``OUSAST_INPUTS`` maps each input's name to
``<producer>/<artifact>``, which the runner fetches after the start from the receiver's ``GET /inputs/...`` to
``OUSAST_INPUT_<NAME>`` under ``/workspace/.ousast-in/<task>`` (the receiver serves a running task its declared
inputs only). The receiver takes the runner's tar (``X-Ousast-Run``, ``X-Ousast-Task``) on
``OUSAST_ARTIFACT_PORT`` (18090), reached by its Service name through the egress gateway; between ``ax apply`` and
resume the task's egress policy is written (``egress.py``).
State: ``~/ousast-results/plane/<run>/`` (``OUSAST_RESULTS``), polled every ``OUSAST_POLL_SECONDS`` (3).
``<producer>/<artifact>``, fetched after the start by the presigned GET of the start body to ``OUSAST_INPUT_<NAME>``
under ``/workspace/.ousast-in/<task>``. Between ``ax apply`` and resume the task's egress policy is written
(``egress.py``). State: ``~/ousast-results/plane/<run>/`` (``OUSAST_RESULTS``), polled every ``OUSAST_POLL_SECONDS``
(3), mirrored to ``runs/<run>/state.json`` on the store after every change and seeded from there by a rerun with no
local state (design section 2); the store is opened first and required, there is no fallback.
"""
from __future__ import annotations
@@ -41,9 +44,12 @@ from urllib.parse import urlsplit
import yaml
from .delivery import Delivery, DeliveryError, declared
from .doctor import doctor
from .egress import RECEIVER_PORT, Egress, Receiver, policy_for
from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests
from .memory import open_store
from .profile import PlaneProfile, load_profile
from .router import Router, credentials, open_router
MODEL_ANNOTATION = "openultrasast.io/model"
@@ -136,15 +142,16 @@ def _model_doc(model: Model) -> dict[str, Any]:
return _doc("Model", model.metadata.name, spec, model.metadata.atespace)
def render_task(run: Run, entry: RunTask, manifests: Manifests, artifact_url: str, dial: str | None = None) -> list[dict[str, Any]]:
def render_task(run: Run, entry: RunTask, manifests: Manifests, artifact_url: str = "", dial: str | None = None) -> list[dict[str, Any]]:
"""The documents ``ax apply`` receives for one Run task: its Model, its Workspaces, the Task last. Task-produced
inputs are never rendered: ax puts every bound Workspace into one env value, which Substrate caps at 32768
characters, so the runner fetches them from the receiver (``OUSAST_INPUTS``) after the start."""
inputs are never rendered (Substrate caps one env value at 32768 characters): the runner fetches them
(``OUSAST_INPUTS``) after the start by presigned URL, or from the receiver when ``artifact_url`` is given."""
task: Task = manifests.tasks[entry.task]
ax_name = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT)
env = {e.name: e.value for e in task.env}
env.update(OUSAST_RUN=run.metadata.name, OUSAST_TASK=entry.name, OUSAST_OUTPUT_DIR=f"{OUTPUT_ROOT}/{entry.name}")
env.update(OUSAST_ARTIFACT_URL=artifact_url, **({"OUSAST_ARTIFACT_DIAL": dial} if dial else {}))
if artifact_url: # receiver mode only (task 2.6 removes it): the store path carries no URL in the env
env.update(OUSAST_ARTIFACT_URL=artifact_url, **({"OUSAST_ARTIFACT_DIAL": dial} if dial else {}))
budget = _fields(entry.budget) if entry.budget else {}
env.update({f"OUSAST_BUDGET_{k.upper()}": str(v) for k, v in budget.items()})
inputs = {re.sub(r"[^A-Z0-9]+", "_", name.upper()): ref for name, ref in entry.inputs.items()}
@@ -224,12 +231,25 @@ class _State:
path: Path
data: dict[str, Any]
lock: threading.Lock = field(default_factory=threading.Lock)
mirror: Callable[[Mapping[str, Any]], None] | None = None # the store copy; a failure is reported, never fatal
def set(self, task: str, **fields: object) -> None:
with self.lock:
self.data["tasks"].setdefault(task, {"status": "pending", "started": None, "finished": None, "usd": None, "model": None})
self.data["tasks"][task].update(fields)
_write_json(self.path, self.data)
self._flush()
def flush(self) -> None:
with self.lock:
self._flush()
def _flush(self) -> None:
_write_json(self.path, self.data)
if self.mirror is not None:
try:
self.mirror(self.data)
except Exception as exc: # noqa: BLE001 - the local file is written; the mirror is reported, the run goes on
print(f"state mirror to the store failed: {exc}", file=sys.stderr)
def status(self, task: str) -> str:
return str(self.data["tasks"].get(task, {}).get("status", "pending"))
@@ -313,25 +333,46 @@ def _await(ax: Ax, name: str, delivered: Callable[[], bool], start: Callable[[],
def _execute(
ax: Ax, egress: Egress, run: Run, entry: RunTask, manifests: Manifests, base: Path, receiver: Receiver, router: Router
ax: Ax,
egress: Egress,
run: Run,
entry: RunTask,
manifests: Manifests,
base: Path,
delivery: Delivery,
router: Router,
receiver: Receiver | None = None,
) -> tuple[str, str]:
ax_name, task = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT), manifests.tasks[entry.task]
atespace, manifest = task.metadata.atespace or "default", base / entry.name / "task.yaml"
manifest.parent.mkdir(parents=True, exist_ok=True)
docs = render_task(run, entry, manifests, receiver.url, receiver.dial)
docs = render_task(run, entry, manifests, receiver.url if receiver else "", receiver.dial if receiver else None)
manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8")
model = manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "")
policy = policy_for(task, [manifests.workspaces[b.name] for b in task.workspaces], model, urlsplit(receiver.url).hostname)
workspaces = [manifests.workspaces[b.name] for b in task.workspaces]
body = None if receiver else delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body
store_host, receiver_host = (None, urlsplit(receiver.url).hostname) if receiver else (delivery.host, None)
if receiver is None:
delivery.reset(entry.name) # a previous attempt's tar must not complete this one
def delivered() -> bool:
return entry.name in receiver.delivered if receiver is not None else delivery.delivered(entry.name)
try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request
start = router.starter(atespace, ax_name, run.metadata.name, credentials(model))
policy = policy_for(task, workspaces, model, store_host, receiver_host)
start = router.starter(atespace, ax_name, run.metadata.name, credentials(model), body)
ax.apply(manifest) # creates the Substrate actor; its egress is denied until the policy exists
egress.apply(ax_name, atespace, policy)
egress.settle()
except RuntimeError as exc: # StartError included: a Model-bound task whose secret is unset
except (RuntimeError, ValueError) as exc: # StartError (a Model's secret unset) and a policy that cannot be granted
ax.delete(ax_name)
return "failed", str(exc)
try:
failure = _await(ax, ax_name, lambda: entry.name in receiver.delivered, start)
failure = _await(ax, ax_name, delivered, start)
if failure is None and receiver is None:
delivery.collect(entry.name, declared(run)[entry.name]) # download, extract, re-publish what is declared
except DeliveryError as exc:
failure = str(exc)
finally:
ax.delete(ax_name)
leftover = egress.delete(ax_name, atespace) # the policy goes with its actor; a survivor is reported
@@ -340,20 +381,44 @@ def _execute(
return ("failed", failure) if failure else outcome_of(_read_json(base / entry.name / "summary.json"))
def run(run_manifest: Path, *, workers: int = 1, ax: str = "ax", results_root: Path | None = None, kubectl_ate: str | None = None) -> str:
"""Execute a Run on ax and return its status: ``done``, ``unfinished`` or ``failed`` (Req 3.1-3.3, 2.2)."""
def run(
run_manifest: Path,
*,
workers: int = 1,
ax: str = "ax",
results_root: Path | None = None,
kubectl_ate: str | None = None,
profile: PlaneProfile | None = None,
) -> str:
"""Execute a Run on ax and return its status: ``done``, ``unfinished`` or ``failed`` (Req 3.1-3.3, 2.2). Every
address comes from ``profile`` (default: the one ``OUSAST_PLANE_PROFILE`` names)."""
profile = profile or load_profile()
run_spec, manifests = load_run(Path(run_manifest))
base = run_dir(run_spec.metadata.name, results_root)
base.mkdir(parents=True, exist_ok=True)
store = open_store(profile.memory) # first, before the lock and before ax sees anything: no fallback
delivery = Delivery(store, run_spec.metadata.name, base)
if not (base / "state.json").exists():
seeded = delivery.load_state() # a rerun from another machine or an empty results root
if seeded is not None:
_write_json(base / "state.json", seeded)
lock = _acquire_lock(base)
state = _State(base / "state.json", _read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}})
port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT)
receiver = Receiver(run_spec.metadata.name, base, int(port)) # the receiver Service targets RECEIVER_PORT
threading.Thread(target=receiver.serve_forever, daemon=True).start()
cli, egress, finished = Ax(ax), Egress(kubectl_ate), queue.Queue[tuple[str, str, str]]()
state = _State(
base / "state.json",
_read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}},
mirror=delivery.save_state,
)
receiver: Receiver | None = None
if os.environ.get("OUSAST_DELIVERY") == "receiver": # the kind profile's host receiver, group 2 only (2.6 removes it)
port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT)
receiver = Receiver(run_spec.metadata.name, base, int(port), profile.kube_context) # the Service targets RECEIVER_PORT
threading.Thread(target=receiver.serve_forever, daemon=True).start()
cli, egress, finished = Ax(ax), Egress(kubectl_ate, profile.kube_context), queue.Queue[tuple[str, str, str]]()
pending = [t for t in run_spec.tasks if state.status(t.name) != "done"]
for entry in run_spec.tasks:
state.set(entry.name, **({"status": "pending"} if entry in pending else {}))
if entry not in pending:
delivery.publish(entry.name, declared(run_spec)[entry.name]) # a seeded or earlier task's artifacts reach its consumers
running: dict[str, RunTask] = {}
halted = False
@@ -361,10 +426,14 @@ def run(run_manifest: Path, *, workers: int = 1, ax: str = "ax", results_root: P
if state.status(entry.name) in ("running", "suspended"):
cli.delete(_ax_name(run_spec.metadata.name, entry.name, limit=TASK_NAME_LIMIT)) # an interrupted attempt; ax must not keep it
state.set(entry.name, status="running", started=_now(), finished=None)
finished.put((entry.name, *_execute(cli, egress, run_spec, entry, manifests, base, receiver, router)))
try:
outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router, receiver)
except Exception as exc: # noqa: BLE001 - a worker that dies silently would hang the run on the queue
outcome = ("failed", f"{type(exc).__name__}: {exc}")
finished.put((entry.name, *outcome))
try:
with open_router() as router: # one port-forward to Substrate's router for the whole run
with open_router(profile.kube_context, url=profile.router_url or None) as router: # one tunnel per run (ax-tunnel)
while pending or running:
blocked = {t.serialize for t in running.values() if t.serialize}
for entry in list(pending):
@@ -375,29 +444,32 @@ def run(run_manifest: Path, *, workers: int = 1, ax: str = "ax", results_root: P
pending.remove(entry)
running[entry.name] = entry
blocked.add(entry.serialize or "")
with receiver.lock: # the receiver serves a running task its declared inputs, nothing else
receiver.running.add(entry.name)
receiver.inputs[entry.name] = frozenset(entry.inputs.values())
if receiver is not None:
with receiver.lock: # the receiver serves a running task its declared inputs, nothing else
receiver.running.add(entry.name)
receiver.inputs[entry.name] = frozenset(entry.inputs.values())
threading.Thread(target=worker, args=(entry,), daemon=True).start()
if not running:
break # nothing can start: a producer stopped short
name, status, reason = finished.get()
del running[name]
with receiver.lock:
receiver.running.discard(name)
receiver.inputs.pop(name, None)
if receiver is not None:
with receiver.lock:
receiver.running.discard(name)
receiver.inputs.pop(name, None)
summary = _read_json(base / name / "summary.json") or {}
state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None)
halted = halted or status == "failed"
if status == "unfinished":
pending = [t for t in pending if name not in t.producers]
finally:
receiver.shutdown()
if receiver is not None:
receiver.shutdown()
lock.unlink(missing_ok=True)
statuses = [state.status(t.name) for t in run_spec.tasks]
result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished"
state.data.update(status=result, finished=_now())
_write_json(state.path, state.data)
state.flush()
return result
+26 -11
View File
@@ -28,6 +28,7 @@ import time
import urllib.error
import urllib.request
from collections.abc import Callable, Iterable, Iterator, Mapping
from typing import Any
from .manifests import Model
@@ -77,7 +78,9 @@ class Router:
def __init__(self, url: str) -> None:
self.url = url.rstrip("/")
def starter(self, atespace: str, task: str, run: str, credentials: Mapping[str, str]) -> Callable[[], str | None]:
def starter(
self, atespace: str, task: str, run: str, credentials: Mapping[str, str], delivery: Mapping[str, Any] | None = None
) -> Callable[[], str | None]:
""":meth:`start_task` bound to one task, returning the refusal text instead of raising (None once accepted).
The reconciler calls it again after a re-resume, since a resumed actor may have rebooted and wait for a
@@ -90,20 +93,30 @@ class Router:
nonlocal calls
calls += 1
try:
self.start_task(atespace, task, run, credentials)
self.start_task(atespace, task, run, credentials, delivery)
except StartError as exc:
return None if calls > 1 and ALREADY_STARTED in str(exc) else str(exc)
return None
return start
def start_task(self, atespace: str, task: str, run: str, credentials: Mapping[str, str]) -> None:
"""Start ``<atespace>/<task>``'s command once; :class:`StartError` on 409, another 4xx, or the timeout."""
body = json.dumps({"run": run, "task": task, "credentials": dict(credentials)}).encode("utf-8")
def start_task(
self, atespace: str, task: str, run: str, credentials: Mapping[str, str], delivery: Mapping[str, Any] | None = None
) -> None:
"""Start ``<atespace>/<task>``'s command once; :class:`StartError` on 409, another 4xx, or the timeout.
``delivery`` (``{put, inputs}`` presigned URLs, ``delivery.py``) rides in the body like the credentials and is
redacted like them."""
payload: dict[str, Any] = {"run": run, "task": task, "credentials": dict(credentials)}
if delivery:
payload["delivery"] = dict(delivery)
body = json.dumps(payload).encode("utf-8")
headers = {"Content-Type": "application/json", TARGET_HEADER: f"{atespace}/{task}"}
limit = float(os.environ.get("OUSAST_START_TIMEOUT") or 300)
deadline, backoff, last = time.monotonic() + limit, 0.5, "no attempt made"
secrets = list(credentials.values())
if delivery:
secrets.append(str(delivery.get("put") or ""))
secrets.extend(str(u) for u in (delivery.get("inputs") or {}).values())
while True:
request = urllib.request.Request(self.url + START_PATH, data=body, method="POST", headers=headers)
try:
@@ -131,18 +144,20 @@ def _drain(stream: Iterable[str], lines: queue.Queue[str | None]) -> None:
@contextlib.contextmanager
def open_router(context: str | None = None, *, kubectl: str = "kubectl", ready_timeout: float = 30.0) -> Iterator[Router]:
def open_router(context: str, *, url: str | None = None, kubectl: str = "kubectl", ready_timeout: float = 30.0) -> Iterator[Router]:
"""A :class:`Router` for the duration of the block.
``OUSAST_ROUTER_URL`` set: that URL, nothing started. Otherwise ``kubectl port-forward svc/atenet-router :80``
in ``ate-system`` of ``context`` (default ``kind-$KIND_CLUSTER_NAME``, ``kind-ousast``), the local port read
from kubectl's ``Forwarding from 127.0.0.1:NNNN`` line; on exit that child process is terminated.
``url`` (the profile's ``router_url``) or ``OUSAST_ROUTER_URL`` set: that URL, nothing started. Otherwise
``kubectl port-forward svc/atenet-router :80`` in ``ate-system`` of the profile's ``context`` (ax-tunnel: the
Kubernetes API is the one authenticated door), the local port read from kubectl's ``Forwarding from
127.0.0.1:NNNN`` line; on exit that child process is terminated. No context is ever built here.
"""
override = os.environ.get("OUSAST_ROUTER_URL")
override = url or os.environ.get("OUSAST_ROUTER_URL")
if override:
yield Router(override)
return
context = context or "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
if not context:
raise StartError("open_router needs the profile's kube context (or router_url)")
argv = [kubectl, "--context", context, "-n", NAMESPACE, "port-forward", SERVICE, ":80"]
proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
try:
+111 -34
View File
@@ -10,18 +10,23 @@ not (ax docs/runner.md).
The command never starts by itself (Requirements 4.5, 4.6). Agent Substrate boots every new ActorTemplate once as a
"golden" actor to snapshot it, with this image and the same ``AX_TASK_YAML``/``AX_WORKSPACES_YAML``/env, and inside
the sandbox nothing tells that boot from the real one. So after the workspaces are ready the runner waits for
``POST /ousast/v1/start`` with ``{"run", "task", "credentials"}``, which the reconciler sends through Substrate's
``POST /ousast/v1/start`` with ``{"run", "task", "credentials", "delivery"}``, which the reconciler sends through Substrate's
router to the task's actor only (``plane/router.py``). ``run`` and ``task`` must match ``OUSAST_RUN`` and the Task's
name (409 otherwise); one request per boot is accepted (202), later ones are 409, and a boot whose completion marker
records a delivered run refuses every request (409 with the reason). Before the workspaces are ready the answer is
503, which the reconciler retries. The credentials live in this process's memory and go into the command's
environment only; they are never logged or written, and their values are redacted from the command's echoed stderr.
``delivery`` (plane-on-kubernetes design section 3) is ``{"put": <presigned PUT URL for the output tar>, "inputs":
{"<NAME>": <presigned GET URL>}}``: a presigned URL is a bearer capability, so it travels where the credentials do.
Each URL must be ``https`` to a hostname (never an address; ``AX_RUNNER_INSECURE_DELIVERY=1`` admits ``http`` in unit
tests only), the answer never echoes one, and no URL is logged.
The command runs as a child process, ``python -m openultrasast.plane.tasks.<spec.command[0]> <spec.command[1:]>``,
in its own process group, with the first workspace as its working directory and ``AX_METADATA_URL`` pointing at
this server. ax has no artifact channel and never reports that a command finished, so after the child exits the
runner reads ``summary.json`` (a crash without one becomes ``failed`` with the exit code and the stderr tail),
posts the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``OUSAST_ARTIFACT_URL`` -- that delivery is the task's
PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` (or POSTs it to ``OUSAST_ARTIFACT_URL``, the receiver
path kept until task 2.6 removes it) -- that delivery is the task's
completion (Requirement 4.4); a failed one is reported ``failed``, never done -- and writes the completion marker
``<OUSAST_STATE_DIR>/<task>.done.json`` (``exit``, ``status``, ``delivered``). Then it keeps serving until SIGTERM,
as ax requires of PID 1. A boot that finds the marker never reruns the command (a resumed actor must not repeat
@@ -38,6 +43,7 @@ Environment the runner reads, all optional except the two ax variables:
AX_RUNNER_AUTOSTART "1" runs the command without waiting for a start request (unit tests only;
required with AX_RUNNER_HTTP=0, where no request could arrive)
AX_RUNNER_TASK_PACKAGE package the command's module is looked up in (unit tests only)
AX_RUNNER_INSECURE_DELIVERY "1" admits http delivery URLs to a hostname (unit tests only; the image never sets it)
OUSAST_STATE_DIR completion markers, default ``/workspace/.ousast-state`` (the durable volume)
OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10
OUSAST_CASE_CACHE a directory of git checkouts (``benchmarks/independent`` cache layout)
@@ -47,8 +53,9 @@ Environment the runner reads, all optional except the two ax variables:
``Host`` (the egress gateway decides on and resolves that name)
OUSAST_DELIVERY_BACKOFF seconds before the second delivery attempt, doubled per retry
OUSAST_INPUTS JSON ``{"<NAME>": "<producer>/<artifact>"}``: after the start and before the
command each is fetched from ``<OUSAST_ARTIFACT_URL>/inputs/<ref>`` (same
dial, headers and retries as delivery) to ``OUSAST_INPUT_<NAME>``; a failed
command each is fetched from ``delivery.inputs[NAME]`` (else from
``<OUSAST_ARTIFACT_URL>/inputs/<ref>`` with the receiver's dial and headers)
to ``OUSAST_INPUT_<NAME>``, with the same retries as delivery; a failed
fetch delivers a failed summary naming the input, the command never runs
The runner exports ``OUSAST_WORKSPACE_DIR`` (the first bound workspace) when the Task's env leaves it unset,
@@ -60,6 +67,7 @@ from __future__ import annotations
import contextlib
import http.client
import io
import ipaddress
import json
import logging
import os
@@ -76,7 +84,7 @@ import urllib.parse
import urllib.request
from collections import deque
from collections.abc import Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any
@@ -206,14 +214,58 @@ _CREDENTIAL_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
_RESERVED_PREFIXES = ("AX_", "OUSAST_") # the runner's own contract; a credential must not redirect it
@dataclass(frozen=True)
class DeliveryTargets:
"""Where this task PUTs its output tar and GETs each input (presigned URLs from the start request)."""
put: str | None = None
inputs: Mapping[str, str] = field(default_factory=dict)
def __repr__(self) -> str: # a presigned URL is a credential: never print one
return f"DeliveryTargets(put={'yes' if self.put else 'no'}, inputs={sorted(self.inputs)})"
@dataclass(frozen=True)
class Start:
"""An accepted start request: the credentials go to the command's environment and nowhere else."""
"""An accepted start request: the credentials go to the command's environment and nowhere else; the delivery
targets go to the artifact PUT and the input GETs and nowhere else."""
credentials: Mapping[str, str]
delivery: DeliveryTargets | None = None
def __repr__(self) -> str: # never print a value by accident
return f"Start(credentials={sorted(self.credentials)})"
return f"Start(credentials={sorted(self.credentials)}, delivery={self.delivery!r})"
def _check_url(url: object, where: str, insecure: bool) -> str:
"""``url`` is a string, ``https`` (``http`` only when ``insecure``) to a hostname that is not an address, without
userinfo; the error names the field and never echoes the value."""
if not isinstance(url, str):
raise ValueError(f"{where} must be a URL string")
parts = urlsplit(url)
allowed = ("https", "http") if insecure else ("https",)
host = parts.hostname or ""
if parts.scheme not in allowed or not host or parts.username is not None or parts.password is not None:
raise ValueError(f"{where} must be an https URL to a hostname")
try:
ipaddress.ip_address(host.strip("[]"))
except ValueError:
return url
raise ValueError(f"{where} must name a host, not an address")
def parse_delivery(raw: object, insecure: bool = False) -> DeliveryTargets | None:
"""The start body's ``delivery`` field validated: None when absent; :class:`ValueError` names the bad field."""
if raw is None:
return None
if not isinstance(raw, dict) or not set(raw) <= {"put", "inputs"}:
raise ValueError("delivery must be an object with put and/or inputs")
put = _check_url(raw["put"], "delivery.put", insecure) if raw.get("put") is not None else None
inputs_raw = raw.get("inputs") or {}
if not isinstance(inputs_raw, dict) or not all(isinstance(k, str) and _CREDENTIAL_NAME.fullmatch(k) for k in inputs_raw):
raise ValueError("delivery.inputs must map ^[A-Z][A-Z0-9_]*$ names to URLs")
inputs = {name: _check_url(url, f"delivery.inputs.{name}", insecure) for name, url in inputs_raw.items()}
return DeliveryTargets(put, inputs)
class StartGate:
@@ -225,7 +277,8 @@ class StartGate:
logged, and no answer echoes it.
"""
def __init__(self) -> None:
def __init__(self, insecure: bool = False) -> None:
self._insecure = insecure # AX_RUNNER_INSECURE_DELIVERY=1: http delivery URLs (unit tests only)
self._lock = threading.Lock()
self._accepted = threading.Event()
self._expected: tuple[str, str] | None = None
@@ -267,10 +320,14 @@ class StartGate:
for k, v in creds.items()
):
return 400, "credentials must map ^[A-Z][A-Z0-9_]*$ names (not AX_/OUSAST_) to strings"
try:
delivery = parse_delivery(request.get("delivery"), self._insecure)
except ValueError as exc:
return 400, str(exc)
run, task = self._expected
if request.get("run") != run or request.get("task") != task:
return 409, f"this actor runs task {task!r} of run {run!r}"
self._start = Start(dict(creds))
self._start = Start(dict(creds), delivery)
self._accepted.set()
return 202, "started"
@@ -653,6 +710,12 @@ def _dialled(url: str, headers: Mapping[str, str], dial: str | None) -> tuple[st
return parts._replace(netloc=dial).geturl(), out
def safe_url(url: str) -> str:
"""``scheme://host/path`` of a URL: what a log line or an error may carry (a presigned query is a credential)."""
parts = urlsplit(url)
return f"{parts.scheme}://{parts.netloc}{parts.path}"
def deliver(
url: str,
payload: bytes,
@@ -661,15 +724,17 @@ def deliver(
attempts: int = DELIVERY_ATTEMPTS,
backoff: float = 1.0,
dial: str | None = None,
method: str = "POST",
) -> int:
"""POST ``payload`` to ``url`` with ``headers``; retry with doubling backoff; return the 2xx status.
"""Send ``payload`` to ``url`` (``POST`` to the receiver, ``PUT`` to a presigned store URL) with ``headers``; retry
with doubling backoff; return the 2xx status. Log lines and errors carry the URL without its query.
``dial`` (``host:port``): connect there instead and send the URL's host as ``Host``, as the actor must -- it
cannot resolve the receiver's cluster name, and the egress gateway decides on ``Host`` and resolves it itself.
"""
last, (target, headers) = "no attempt made", _dialled(url, headers, dial)
for attempt in range(1, attempts + 1):
request = urllib.request.Request(target, data=payload, method="POST", headers=headers)
request = urllib.request.Request(target, data=payload, method=method, headers=headers)
try:
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the reconciler's URL
status = int(response.status)
@@ -680,10 +745,10 @@ def deliver(
last = f"HTTP {exc.code}"
except (urllib.error.URLError, http.client.HTTPException, OSError, ValueError) as exc:
last = f"{type(exc).__name__}: {exc}"
log.warning("artifact delivery attempt %d/%d to %s failed: %s", attempt, attempts, url, last)
log.warning("artifact delivery attempt %d/%d to %s failed: %s", attempt, attempts, safe_url(url), last)
if attempt < attempts:
time.sleep(backoff * (2 ** (attempt - 1)))
raise DeliveryError(f"{last} after {attempts} attempts to {url}")
raise DeliveryError(f"{last} after {attempts} attempts to {safe_url(url)}")
def fetch(
@@ -711,14 +776,15 @@ def fetch(
except (urllib.error.URLError, http.client.HTTPException, OSError, ValueError) as exc:
last = f"{type(exc).__name__}: {exc}"
partial.unlink(missing_ok=True)
log.warning("input fetch attempt %d/%d from %s failed: %s", attempt, attempts, url, last)
log.warning("input fetch attempt %d/%d from %s failed: %s", attempt, attempts, safe_url(url), last)
if attempt < attempts:
time.sleep(backoff * (2 ** (attempt - 1)))
raise DeliveryError(f"{last} after {attempts} attempts to {url}")
raise DeliveryError(f"{last} after {attempts} attempts to {safe_url(url)}")
def fetch_inputs(environ: Mapping[str, str]) -> list[Path]:
"""Fetch every ``OUSAST_INPUTS`` entry from the receiver to its ``OUSAST_INPUT_<NAME>`` path.
def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> list[Path]:
"""Fetch every ``OUSAST_INPUTS`` entry to its ``OUSAST_INPUT_<NAME>`` path: from ``delivery.inputs[NAME]`` (a
presigned GET, no headers) when the start request carries one, else from the receiver.
:class:`RunnerError` names the input when one cannot be fetched: a task never runs with an input missing."""
raw = environ.get("OUSAST_INPUTS")
@@ -736,11 +802,15 @@ def fetch_inputs(environ: Mapping[str, str]) -> list[Path]:
fetched: list[Path] = []
for name, ref in sorted(inputs.items()):
dest = environ.get(f"OUSAST_INPUT_{name}")
if not url or not dest or not isinstance(ref, str):
raise RunnerError(f"input {name} ({ref}) not fetched: OUSAST_ARTIFACT_URL or OUSAST_INPUT_{name} is unset")
source = f"{url.rstrip('/')}/inputs/{urllib.parse.quote(ref)}"
presigned = delivery.inputs.get(name) if delivery is not None else None
if not dest or not isinstance(ref, str) or not (presigned or url):
raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL, OUSAST_ARTIFACT_URL or OUSAST_INPUT_{name}")
try:
size = fetch(source, headers, Path(dest), backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
if presigned:
size = fetch(presigned, {}, Path(dest), backoff=backoff)
else:
source = f"{str(url).rstrip('/')}/inputs/{urllib.parse.quote(ref)}"
size = fetch(source, headers, Path(dest), backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
except DeliveryError as exc:
raise RunnerError(f"input {name} ({ref}) not fetched: {exc}") from None
log.info("input %s (%s): %d bytes to %s", name, ref, size, dest)
@@ -748,8 +818,13 @@ def fetch_inputs(environ: Mapping[str, str]) -> list[Path]:
return fetched
def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str]) -> bool:
"""POST the output directory; False when there is nowhere to deliver; :class:`DeliveryError` when it fails."""
def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool:
"""PUT the output tar to ``delivery.put`` when the start request carried one, else POST it to the receiver;
False when there is nowhere to deliver; :class:`DeliveryError` when it fails."""
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
if delivery is not None and delivery.put:
deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff, method="PUT")
return True
url = environ.get("OUSAST_ARTIFACT_URL")
if not url:
return False
@@ -758,7 +833,6 @@ def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str]) ->
run = environ.get("OUSAST_RUN")
if run:
headers["X-Ousast-Run"] = run
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
deliver(url, build_tar(output_dir), headers, backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
return True
@@ -810,14 +884,16 @@ def _exit_of(marker: Mapping[str, Any]) -> int:
return EXIT_BY_STATUS.get(str(marker.get("status")), EXIT_BY_STATUS["failed"])
def _finish(session: _Session, env: Mapping[str, str], exit_code: int | None, summary: Mapping[str, Any]) -> dict[str, Any]:
def _finish(
session: _Session, env: Mapping[str, str], exit_code: int | None, summary: Mapping[str, Any], delivery: DeliveryTargets | None = None
) -> dict[str, Any]:
"""Deliver the output directory and write the completion marker; a failed delivery reports ``failed``.
The marker keeps the task's own summary so a later boot can put it back and retry the delivery alone.
"""
marker: dict[str, Any] = {"exit": exit_code, "status": str(summary.get("status")), "delivered": False, "summary": dict(summary)}
try:
marker["delivered"] = _deliver_output(session.task, session.output_dir, env)
marker["delivered"] = _deliver_output(session.task, session.output_dir, env, delivery)
except DeliveryError as exc:
log.error("artifact delivery failed: %s", exc)
marker.update(status="failed", reason=f"artifact delivery failed: {exc}")
@@ -857,24 +933,24 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa
stored = done.get("summary")
kept: dict[str, Any] = stored if isinstance(stored, dict) else read_summary(session.output_dir) or {}
write_summary(session.output_dir, kept)
return _exit_of(_finish(session, env, done.get("exit"), kept))
return _exit_of(_finish(session, env, done.get("exit"), kept, start.delivery)) # a re-minted URL, if any
if stop.is_set():
return 0
try:
_materialise_once(session, env)
fetch_inputs(env) # after the start as well: the receiver serves a task its inputs only while it runs
fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it (and a receiver serves a running task only)
except RunnerError as exc:
log.error("runner failed: %s", exc)
failed = _failed(f"runner failed: {exc}", read_summary(session.output_dir))
write_summary(session.output_dir, failed)
return _exit_of(_finish(session, env, None, failed))
return _exit_of(_finish(session, env, None, failed, start.delivery))
log.info("workspaces and inputs ready; running task %s: %s", session.task.metadata.name, list(session.task.command))
grace = float(env.get("OUSAST_TERM_GRACE") or TERM_GRACE)
code, summary = run_command(session.task, session.output_dir, env, stop, grace, start.credentials)
if code is None:
log.info("task %s was stopped before its command finished; a resume runs it again", session.task.metadata.name)
return 0
return _exit_of(_finish(session, env, code, summary))
return _exit_of(_finish(session, env, code, summary, start.delivery))
except RunnerError as exc:
# The failure is reported, never acted on unasked: a start request is answered with it (409 without a
# session, else 202 and the failed summary is delivered), so the golden boot delivers nothing either.
@@ -887,9 +963,10 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa
write_summary(session.output_dir, summary)
gate.arm(env.get("OUSAST_RUN", ""), session.task.metadata.name)
ready.set()
if env.get("AX_RUNNER_AUTOSTART") != "1" and gate.wait(stop) is None:
late = gate.autostart() if env.get("AX_RUNNER_AUTOSTART") == "1" else gate.wait(stop)
if late is None:
return 0
return _exit_of(_finish(session, env, None, summary))
return _exit_of(_finish(session, env, None, summary, late.delivery))
def _materialise_once(session: _Session, env: Mapping[str, str]) -> None:
@@ -918,14 +995,14 @@ def main(argv: Sequence[str] | None = None, environ: MutableMapping[str, str] |
env = os.environ if environ is None else environ
if argv:
log.warning("ax-task-runner ignores command-line arguments %r; the Task manifest is the only input", list(argv))
ready, stop, gate = threading.Event(), threading.Event(), StartGate()
ready, stop, gate = threading.Event(), threading.Event(), StartGate(insecure=env.get("AX_RUNNER_INSECURE_DELIVERY") == "1")
previous = _on_sigterm(stop)
server: HealthServer | None = None
if env.get("AX_RUNNER_HTTP", "1") != "0":
metadata = {"task": env.get("AX_TASK_YAML", ""), "workspaces": env.get("AX_WORKSPACES_YAML", "")}
server = start_health_server(int(env.get("AX_RUNNER_PORT", str(DEFAULT_PORT))), ready, metadata, gate)
env["AX_RUNNER_BOUND_PORT"] = str(server.port)
env["AX_METADATA_URL"] = f"http://127.0.0.1:{server.port}"
env["AX_METADATA_URL"] = f"http://127.0.0.1:{server.port}" # loopback: this runner's own health server
log.info("health and metadata server on port %d", server.port)
try:
if server is None and env.get("AX_RUNNER_AUTOSTART") != "1":