mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
feat(plane): router-addressed start signal; credentials only in the start request
Agent Substrate's golden-snapshot boot runs the same image with the same task env, so the runner no longer starts its command by itself: after the workspaces are ready it waits for POST /ousast/v1/start (run/task checked, once per boot, refused after a delivered run). The reconciler sends it through svc/atenet-router with ate-target-actor, which never addresses the golden actor (plane/router.py), and the bound Model's secretKey variable travels only in that body, into the child's env. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
ac296ee3a4
commit
14ea48b1ea
@@ -0,0 +1,170 @@
|
||||
"""The start signal through Agent Substrate's router (ai-service-plane Requirements 4.5, 4.6).
|
||||
|
||||
Agent Substrate boots every new ActorTemplate once as a temporary "golden" actor to capture a snapshot, with the
|
||||
same image and the same task environment ax baked into the template, so a runner cannot tell that boot from the
|
||||
real one. The runner therefore never starts its command by itself: it prepares the workspaces, reports ready and
|
||||
waits for ``POST /ousast/v1/start``. The reconciler sends that request through Substrate's router
|
||||
(``svc/atenet-router`` in ``ate-system``), which routes on ``ate-target-actor: <atespace>/<task>`` to the task's
|
||||
actor only -- never to the golden one -- and resumes a suspended actor before proxying.
|
||||
|
||||
:func:`open_router` port-forwards the router to a random local port for the duration of a Run (or uses
|
||||
``OUSAST_ROUTER_URL`` when set, as the tests do); :meth:`Router.start_task` posts the start body, retrying 502/503/504
|
||||
and connection errors with backoff for up to ``OUSAST_START_TIMEOUT`` s (default 300). Provider credentials travel
|
||||
only in that body: :func:`credentials` reads the variable a bound Model's ``secretKey`` names from this process's
|
||||
environment (``ousast`` loads ``.env`` at startup); nothing here writes or logs a value.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping
|
||||
|
||||
from .manifests import Model
|
||||
|
||||
__all__ = ["ALREADY_STARTED", "START_PATH", "Router", "StartError", "credentials", "open_router", "redact"]
|
||||
|
||||
START_PATH = "/ousast/v1/start"
|
||||
ALREADY_STARTED = "already started on this boot" # the runner's 409 when its command is running
|
||||
NAMESPACE = "ate-system"
|
||||
SERVICE = "svc/atenet-router"
|
||||
TARGET_HEADER = "ate-target-actor"
|
||||
RETRY_STATUS = frozenset({502, 503, 504})
|
||||
_FORWARDING = re.compile(r"Forwarding from 127\.0\.0\.1:(\d+)")
|
||||
|
||||
|
||||
class StartError(RuntimeError):
|
||||
"""The start request was refused, or no actor answered it in time; the message carries the response text."""
|
||||
|
||||
|
||||
def redact(text: str, secrets: Iterable[str]) -> str:
|
||||
"""``text`` with every non-empty secret value replaced by ``[redacted]``."""
|
||||
for secret in secrets:
|
||||
if secret:
|
||||
text = text.replace(secret, "[redacted]")
|
||||
return text
|
||||
|
||||
|
||||
def credentials(model: Model | None, environ: Mapping[str, str] | None = None) -> dict[str, str]:
|
||||
"""The provider credentials of a task bound to ``model``: ``{secretKey.key: value}`` from the environment.
|
||||
|
||||
A task without a Model, or a Model without ``secretKey``, gets none. A named variable that is unset or empty
|
||||
raises :class:`StartError` naming the variable (never a value).
|
||||
"""
|
||||
if model is None or model.secret_key is None:
|
||||
return {}
|
||||
env = os.environ if environ is None else environ
|
||||
name = model.secret_key.key
|
||||
value = env.get(name, "")
|
||||
if not value:
|
||||
where = f"secretKey {model.secret_key.name}; it is not set in this environment or .env"
|
||||
raise StartError(f"Model/{model.metadata.name} needs {name} ({where})")
|
||||
return {name: value}
|
||||
|
||||
|
||||
class Router:
|
||||
"""Posts start requests to one task actor at a time through the router at ``url``."""
|
||||
|
||||
def __init__(self, url: str) -> None:
|
||||
self.url = url.rstrip("/")
|
||||
|
||||
def starter(self, atespace: str, task: str, run: str, credentials: Mapping[str, str]) -> Callable[[], str | None]:
|
||||
""":meth:`start_task` bound to one task, returning the refusal text instead of raising (None once accepted).
|
||||
|
||||
The reconciler calls it again after a re-resume, since a resumed actor may have rebooted and wait for a
|
||||
start; on those later calls a 409 saying the command is already started on this boot means the process
|
||||
tree survived and the command is still running, which is not a failure.
|
||||
"""
|
||||
calls = 0
|
||||
|
||||
def start() -> str | None:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
try:
|
||||
self.start_task(atespace, task, run, credentials)
|
||||
except StartError as exc:
|
||||
return None if calls > 1 and ALREADY_STARTED in str(exc) else str(exc)
|
||||
return None
|
||||
|
||||
return start
|
||||
|
||||
def start_task(self, atespace: str, task: str, run: str, credentials: Mapping[str, str]) -> None:
|
||||
"""Start ``<atespace>/<task>``'s command once; :class:`StartError` on 409, another 4xx, or the timeout."""
|
||||
body = json.dumps({"run": run, "task": task, "credentials": dict(credentials)}).encode("utf-8")
|
||||
headers = {"Content-Type": "application/json", TARGET_HEADER: f"{atespace}/{task}"}
|
||||
limit = float(os.environ.get("OUSAST_START_TIMEOUT") or 300)
|
||||
deadline, backoff, last = time.monotonic() + limit, 0.5, "no attempt made"
|
||||
secrets = list(credentials.values())
|
||||
while True:
|
||||
request = urllib.request.Request(self.url + START_PATH, data=body, method="POST", headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the router's local URL
|
||||
if 200 <= int(response.status) < 300:
|
||||
return
|
||||
last = f"HTTP {response.status}"
|
||||
except urllib.error.HTTPError as exc:
|
||||
text = redact(exc.read().decode("utf-8", errors="replace").strip()[:400], secrets)
|
||||
if exc.code not in RETRY_STATUS:
|
||||
raise StartError(f"start of {atespace}/{task} refused (HTTP {exc.code}): {text}") from None
|
||||
last = f"HTTP {exc.code}: {text}"
|
||||
except (urllib.error.URLError, http.client.HTTPException, OSError) as exc:
|
||||
last = redact(f"{type(exc).__name__}: {exc}", secrets)
|
||||
if time.monotonic() + backoff > deadline:
|
||||
raise StartError(f"start of {atespace}/{task} not accepted within {limit:g}s (OUSAST_START_TIMEOUT): {last}")
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, 10.0)
|
||||
|
||||
|
||||
def _drain(stream: Iterable[str], lines: queue.Queue[str | None]) -> None:
|
||||
for line in stream: # kubectl logs a line per proxied connection; an unread pipe would block it
|
||||
lines.put(line)
|
||||
lines.put(None)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def open_router(context: str | None = None, *, kubectl: str = "kubectl", ready_timeout: float = 30.0) -> Iterator[Router]:
|
||||
"""A :class:`Router` for the duration of the block.
|
||||
|
||||
``OUSAST_ROUTER_URL`` set: that URL, nothing started. Otherwise ``kubectl port-forward svc/atenet-router :80``
|
||||
in ``ate-system`` of ``context`` (default ``kind-$KIND_CLUSTER_NAME``, ``kind-ousast``), the local port read
|
||||
from kubectl's ``Forwarding from 127.0.0.1:NNNN`` line; on exit that child process is terminated.
|
||||
"""
|
||||
override = os.environ.get("OUSAST_ROUTER_URL")
|
||||
if override:
|
||||
yield Router(override)
|
||||
return
|
||||
context = context or "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
|
||||
argv = [kubectl, "--context", context, "-n", NAMESPACE, "port-forward", SERVICE, ":80"]
|
||||
proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
try:
|
||||
assert proc.stdout is not None
|
||||
lines: queue.Queue[str | None] = queue.Queue()
|
||||
threading.Thread(target=_drain, args=(proc.stdout, lines), name="router-port-forward", daemon=True).start()
|
||||
deadline, seen, port = time.monotonic() + ready_timeout, list[str](), None
|
||||
while port is None:
|
||||
try:
|
||||
line = lines.get(timeout=max(deadline - time.monotonic(), 0.01))
|
||||
except queue.Empty:
|
||||
raise StartError(f"kubectl port-forward printed no port within {ready_timeout:g}s: {''.join(seen)[-400:]}") from None
|
||||
if line is None:
|
||||
raise StartError(f"kubectl port-forward exited ({proc.wait()}): {''.join(seen)[-400:]}")
|
||||
seen.append(line)
|
||||
match = _FORWARDING.search(line)
|
||||
port = int(match.group(1)) if match else None
|
||||
yield Router(f"http://127.0.0.1:{port}")
|
||||
finally:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
Reference in New Issue
Block a user