feat(plane): router-addressed start signal; credentials only in the start request

Agent Substrate's golden-snapshot boot runs the same image with the same task env,
so the runner no longer starts its command by itself: after the workspaces are
ready it waits for POST /ousast/v1/start (run/task checked, once per boot, refused
after a delivered run). The reconciler sends it through svc/atenet-router with
ate-target-actor, which never addresses the golden actor (plane/router.py), and the
bound Model's secretKey variable travels only in that body, into the child's env.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-29 19:41:35 +02:00
co-authored by Claude Opus 5.5
parent ac296ee3a4
commit 14ea48b1ea
10 changed files with 919 additions and 95 deletions
+12 -1
View File
@@ -3,11 +3,13 @@
The tests copy it into a throwaway package (``AX_RUNNER_TASK_PACKAGE``) so the runner starts it the way it starts
a real task, ``python -m <package>.stub_task``. ``STUB_MODE`` picks the outcome (``done``, ``failed``,
``unfinished``, ``crash``, ``nosummary``, ``exit``, ``sleep``); ``STUB_COUNTER`` names a file that gets one line
per run.
per run. ``STUB_SECRET_VAR`` names a credential variable: the stub records the SHA-256 of its value (or None),
never the value.
"""
from __future__ import annotations
import hashlib
import json
import os
import signal
@@ -23,6 +25,11 @@ def _get(url: str) -> tuple[int, str]:
return int(response.status), response.read().decode("utf-8")
def _secret_hash(name: str | None) -> str | None:
value = os.environ.get(name) if name else None
return hashlib.sha256(value.encode("utf-8")).hexdigest() if value else None
def main(argv: list[str]) -> None:
out = Path(os.environ["OUSAST_OUTPUT_DIR"])
out.mkdir(parents=True, exist_ok=True)
@@ -45,6 +52,7 @@ def main(argv: list[str]) -> None:
"pgid": os.getpgid(0),
"metadata_url": metadata,
"metadata_task": _get(f"{metadata}/metadata/v1alpha1/ax/task")[1] if metadata else None,
"secret_sha256": _secret_hash(os.environ.get("STUB_SECRET_VAR")),
}
(out / "facts.json").write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8")
if mode == "sleep": # ignore SIGTERM so only the runner's SIGKILL ends the group; the grandchild inherits it
@@ -52,6 +60,9 @@ def main(argv: list[str]) -> None:
grandchild = subprocess.Popen(["sleep", "60"])
(out / "pids.json").write_text(json.dumps([os.getpid(), grandchild.pid]), encoding="utf-8")
time.sleep(60)
if mode == "leak": # a task that prints its credential and crashes: the runner must redact it
sys.stderr.write(f"auth failed for key {os.environ.get(os.environ.get('STUB_SECRET_VAR', ''), '')}\n")
raise SystemExit(5)
if mode == "crash":
raise RuntimeError("stub exploded")
if mode == "exit":
+108 -3
View File
@@ -5,7 +5,8 @@ internal/controller/reconciler.go, internal/server/server.go): ``apply`` creates
fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new
image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase
-- and on the scripted poll it POSTs a scripted tar to the ``OUSAST_ARTIFACT_URL`` in the applied Task's env, or
turns ``Failed`` with a condition message, or vanishes. Every outcome comes from a scripted delivery.
turns ``Failed`` with a condition message, or vanishes. Every outcome comes from a scripted delivery, and a
delivery happens only after the reconciler's start request reached the fake router (Req 4.5).
"""
from __future__ import annotations
@@ -16,7 +17,10 @@ import stat
import subprocess
import sys
import tarfile
import threading
import time
from collections.abc import Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
@@ -27,6 +31,7 @@ from openultrasast.plane.workspaces import workspaces
RECONCILER = Path("src/openultrasast/plane/reconciler.py")
PIN = "0123456789abcdef0123456789abcdef01234567"
SECRET = "sk-reconciler-test-7a2b-only-in-the-start-request"
# The fields ax's API server decodes (pkg/apis/v1alpha1/ax.proto, protojson: an unknown field is an error at
# ``ax apply``). ``None`` marks a value ax types as free-form (a Struct) or that this test does not descend into.
@@ -114,6 +119,9 @@ elif argv[:2] == ["get", "task"]:
rec.update(phase=script["phase"], message=script.get("message", ""))
if script.get("vanish"):
rec["gone"] = True
started = (HERE / "started" / argv[2]).exists() # the runner runs its command only after the start request
if rec["phase"] == "Running" and rec["polls"] >= script.get("polls", 1) and started and not rec.get("sent"):
rec["sent"] = True
if script.get("deliver", True) and "phase" not in script and not script.get("vanish"):
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode="w") as tar:
@@ -152,6 +160,7 @@ metadata:
spec:
provider: deepseek
model: deepseek-chat
secretKey: {name: deepseek, key: DEEPSEEK_API_KEY}
parameters: {cache_hit_per_m: 0.07, input_per_m: 0.27, output_per_m: 1.1}
---
apiVersion: ax.io/v1alpha1
@@ -234,6 +243,8 @@ SCRIPT = {
class Fake:
router: FakeRouter
def __init__(self, root: Path) -> None:
self.root = root
self.ax = root / "ax"
@@ -263,15 +274,61 @@ class Fake:
return out
class FakeRouter(ThreadingHTTPServer):
"""Agent Substrate's router as the reconciler sees it: records each start request and answers from ``codes``
(by ax task name, default 202); an accepted start marks the task started for the fake ax, whose runner only
then runs its command and delivers."""
daemon_threads = True
def __init__(self, root: Path) -> None:
super().__init__(("127.0.0.1", 0), _RouterHandler)
self.root = root
self.codes: dict[str, tuple[int, str]] = {}
self.starts: list[dict] = []
threading.Thread(target=self.serve_forever, daemon=True).start()
class _RouterHandler(BaseHTTPRequestHandler):
server: FakeRouter
def do_POST(self) -> None: # noqa: N802
body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)))
target = self.headers.get("ate-target-actor", "")
name = target.partition("/")[2]
code, text = self.server.codes.get(name, (202, "started"))
self.server.starts.append({"at": time.monotonic(), "path": self.path, "target": target, "body": body, "code": code})
if code == 202:
(self.server.root / "started").mkdir(exist_ok=True)
(self.server.root / "started" / name).touch()
with (self.server.root / "ax.log").open("a") as log:
log.write(f"{time.monotonic():.4f} started {name} {code}\n")
self.send_response(code)
self.send_header("Content-Length", str(len(text.encode())))
self.end_headers()
self.wfile.write(text.encode())
def log_message(self, format: str, *args: object) -> None: # noqa: A002
pass
@pytest.fixture
def fake(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Fake:
def fake(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Fake]:
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path / "results"))
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
monkeypatch.setenv("OUSAST_POLL_SECONDS", "0.02")
monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "20") # a test that never delivers fails in seconds, not in 2 hours
monkeypatch.setenv("OUSAST_START_TIMEOUT", "2")
monkeypatch.setenv("DEEPSEEK_API_KEY", SECRET) # the Model's secretKey variable, as ``.env`` would set it
fake = Fake(tmp_path)
fake.script(SCRIPT)
return fake
fake.router = FakeRouter(tmp_path)
monkeypatch.setenv("OUSAST_ROUTER_URL", f"http://127.0.0.1:{fake.router.server_address[1]}")
try:
yield fake
finally:
fake.router.shutdown()
fake.router.server_close()
def write_run(tmp_path: Path, name: str, tasks: str = CHAIN) -> Path:
@@ -426,6 +483,54 @@ def test_no_delivery_within_the_task_timeout_is_failed(fake: Fake, tmp_path: Pat
assert [n for _, n in fake.events("delete")] == ["silent-repo-facts"], "ax delete follows every task"
def files_containing(root: Path, needle: str) -> list[str]:
return sorted(str(p) for p in root.rglob("*") if p.is_file() and needle.encode() in p.read_bytes())
def test_start_is_sent_once_after_resume_with_the_models_credential(fake: Fake, tmp_path: Path) -> None:
"""Req 4.5/4.6: one start per task through the router, after ax accepted a resume; the bound Model's credential
travels only in that request, never in the rendered Task, the run dir or any file the run leaves behind."""
assert reconciler.run(write_run(tmp_path, "sig"), ax=str(fake.ax)) == "done"
starts = fake.router.starts
assert [s["target"] for s in starts] == ["default/sig-repo-facts", "default/sig-verify", "default/sig-agree"], "once each"
assert all(s["path"] == "/ousast/v1/start" and s["code"] == 202 for s in starts)
by_task = {s["body"]["task"]: s for s in starts}
assert by_task["sig-verify"]["body"] == {"run": "sig", "task": "sig-verify", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
assert by_task["sig-repo-facts"]["body"]["credentials"] == {} and by_task["sig-agree"]["body"]["credentials"] == {}, "no Model"
accepted = {}
for stamp, words in fake.log(): # the last resume of each task is the one ax accepted
if words[:2] == ["resume", "task"]:
accepted[words[2]] = stamp
delivered = {name: stamp for stamp, name in fake.events("delivered")}
for s in starts:
name = s["body"]["task"]
assert accepted[name] < s["at"] < delivered[name.removeprefix("sig-")], f"{name}: resume, then start, then delivery"
verify_model = fake.applied("sig", "verify")["docs"][1]
assert verify_model["spec"]["secretKey"] == {"name": "deepseek", "key": "DEEPSEEK_API_KEY"}, "the manifest names the key"
assert "DEEPSEEK_API_KEY" not in fake.applied("sig", "verify")["env"], "the rendered Task carries no credential"
rendered = str(reconciler.run_dir("sig") / "verify" / "task.yaml")
assert rendered in files_containing(tmp_path, "OUSAST_ARTIFACT_URL"), "the scan reads the rendered files it vouches for"
assert files_containing(tmp_path, SECRET) == [], "no rendered task.yaml, state, log or applied record holds the value"
def test_a_model_bound_task_without_its_secret_fails_before_ax_sees_it(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("DEEPSEEK_API_KEY")
assert reconciler.run(write_run(tmp_path, "nokey"), ax=str(fake.ax)) == "failed"
assert statuses("nokey") == {"repo-facts": "done", "verify": "failed", "agree": "pending"}
reason = state_of("nokey")["tasks"]["verify"]["reason"]
assert "DEEPSEEK_API_KEY" in reason and "deepseek-flash" in reason
assert [name for _, name in fake.events("apply")] == ["repo-facts"], "verify is never applied without its credential"
def test_a_refused_start_fails_the_task_with_the_runners_answer(fake: Fake, tmp_path: Path) -> None:
fake.router.codes["ref-repo-facts"] = (409, "task ref-repo-facts already ran and was delivered (done)")
assert reconciler.run(write_run(tmp_path, "ref"), ax=str(fake.ax)) == "failed"
reason = state_of("ref")["tasks"]["repo-facts"]["reason"]
assert "HTTP 409" in reason and "already ran and was delivered" in reason
assert len(fake.router.starts) == 1, "a refusal is final, not retried"
assert [n for _, n in fake.events("delete")] == ["ref-repo-facts"], "ax delete follows a refused start"
def test_outcome_of_needs_a_delivered_summary() -> None:
assert reconciler.outcome_of(None) == ("failed", "no summary.json delivered")
assert reconciler.outcome_of({"status": "done", "units_done": 1, "units_total": 1}) == ("done", "")
+189
View File
@@ -0,0 +1,189 @@
"""The start signal through Agent Substrate's router (ai-service-plane Requirements 4.5, 4.6): port-forward
lifecycle with a fake kubectl, retry and refusal against a fake router, credentials from the environment."""
from __future__ import annotations
import json
import stat
import sys
import threading
import time
from collections.abc import Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from openultrasast.plane.manifests import Metadata, Model, SecretKey
from openultrasast.plane.router import START_PATH, StartError, credentials, open_router
SECRET = "sk-router-test-5d1e-never-logged"
FAKE_KUBECTL = """#!{python}
import json, os, sys, time
from pathlib import Path
here = Path(__file__).resolve().parent
(here / "kubectl.json").write_text(json.dumps({{"argv": sys.argv[1:], "pid": os.getpid()}}))
mode = os.environ.get("FAKE_KUBECTL_MODE", "ok")
if mode == "fail":
print('error: services "atenet-router" not found', flush=True)
sys.exit(1)
print("Forwarding from 127.0.0.1:{port} -> 80", flush=True)
print("Forwarding from [::1]:{port} -> 80", flush=True)
time.sleep(600)
"""
class FakeRouter(ThreadingHTTPServer):
"""Answers the start path from ``codes`` in turn (the last one repeats) and records every request."""
daemon_threads = True
def __init__(self, codes: list[tuple[int, str]]) -> None:
super().__init__(("127.0.0.1", 0), _RouterHandler)
self.codes = codes
self.requests: list[tuple[str, dict[str, str], dict]] = []
threading.Thread(target=self.serve_forever, daemon=True).start()
@property
def url(self) -> str:
return f"http://127.0.0.1:{self.server_address[1]}"
class _RouterHandler(BaseHTTPRequestHandler):
server: FakeRouter
def do_POST(self) -> None: # noqa: N802
body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)))
self.server.requests.append((self.path, {k.lower(): v for k, v in self.headers.items()}, body))
code, text = self.server.codes[min(len(self.server.requests), len(self.server.codes)) - 1]
payload = text.encode()
self.send_response(code)
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def log_message(self, format: str, *args: object) -> None: # noqa: A002
pass
@pytest.fixture
def fake_router(request: pytest.FixtureRequest) -> Iterator[FakeRouter]:
server = FakeRouter(getattr(request, "param", [(202, "started")]))
try:
yield server
finally:
server.shutdown()
server.server_close()
def write_kubectl(tmp_path: Path, port: int) -> Path:
path = tmp_path / "kubectl"
path.write_text(FAKE_KUBECTL.format(python=sys.executable, port=port), encoding="utf-8")
path.chmod(path.stat().st_mode | stat.S_IXUSR)
return path
def alive(pid: int) -> bool:
try:
return Path(f"/proc/{pid}/stat").read_text().split(")")[-1].split()[0] != "Z"
except (FileNotFoundError, IndexError):
return False
@pytest.fixture(autouse=True)
def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None:
for key in ("OUSAST_ROUTER_URL", "OUSAST_START_TIMEOUT", "KIND_CLUSTER_NAME", "FAKE_KUBECTL_MODE"):
monkeypatch.delenv(key, raising=False)
def test_port_forward_port_is_parsed_and_the_child_is_terminated_on_exit(tmp_path: Path, fake_router: FakeRouter) -> None:
port = int(fake_router.server_address[1])
kubectl = write_kubectl(tmp_path, port)
with open_router(kubectl=str(kubectl)) as router:
assert router.url == f"http://127.0.0.1:{port}"
seen = json.loads((tmp_path / "kubectl.json").read_text())
assert seen["argv"] == ["--context", "kind-ousast", "-n", "ate-system", "port-forward", "svc/atenet-router", ":80"]
assert alive(seen["pid"])
router.start_task("default", "run-t", "run", {})
deadline = time.monotonic() + 5
while alive(seen["pid"]) and time.monotonic() < deadline:
time.sleep(0.05)
assert not alive(seen["pid"]), "the port-forward child is terminated when the block ends"
assert fake_router.requests[0][1]["ate-target-actor"] == "default/run-t"
def test_a_failing_port_forward_raises_with_kubectls_output(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("FAKE_KUBECTL_MODE", "fail")
kubectl = write_kubectl(tmp_path, 1)
with pytest.raises(StartError, match="atenet-router.*not found"), open_router("kind-other", kubectl=str(kubectl)):
pass
assert json.loads((tmp_path / "kubectl.json").read_text())["argv"][1] == "kind-other"
def test_router_url_override_starts_no_port_forward(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
with open_router(kubectl=str(tmp_path / "no-such-kubectl")) as router:
assert router.url == fake_router.url
assert not (tmp_path / "kubectl.json").exists()
@pytest.mark.parametrize("fake_router", [[(503, "no actor yet"), (502, "bad gateway"), (202, "started")]], indirect=True)
def test_start_retries_503_and_502_then_succeeds(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
with open_router() as router:
router.start_task("default", "chain-verify", "chain", {"DEEPSEEK_API_KEY": SECRET})
assert len(fake_router.requests) == 3
path, headers, body = fake_router.requests[-1]
assert path == START_PATH and headers["ate-target-actor"] == "default/chain-verify"
assert body == {"run": "chain", "task": "chain-verify", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
@pytest.mark.parametrize("fake_router", [[(409, f"this actor runs task 'x' (echo {SECRET})")]], indirect=True)
def test_a_409_is_raised_at_once_with_the_response_text_redacted(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
with open_router() as router, pytest.raises(StartError, match="HTTP 409.*runs task 'x'") as caught:
router.start_task("default", "t", "r", {"DEEPSEEK_API_KEY": SECRET})
assert len(fake_router.requests) == 1 and SECRET not in str(caught.value)
assert router.starter("default", "t", "r", {})() is not None, "the starter returns the refusal instead of raising"
@pytest.mark.parametrize("fake_router", [[(409, "this task was already started on this boot")]], indirect=True)
def test_a_restart_after_re_resume_accepts_a_running_command(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
"""First start: 'already started' is a refusal. After a re-resume the actor may not have rebooted; then the
runner's 'already started on this boot' means the command still runs, not a failure."""
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
with open_router() as router:
start = router.starter("default", "t", "r", {})
assert start() is not None, "a first start that finds the command running is a refusal"
assert start() is None, "a re-sent start after a re-resume is not"
fake_router.codes = [(409, "task t already ran and was delivered (done)")]
assert start() is not None, "any other refusal stays one"
@pytest.mark.parametrize("fake_router", [[(503, "workspaces not materialised")]], indirect=True)
def test_start_gives_up_after_the_start_timeout(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
monkeypatch.setenv("OUSAST_START_TIMEOUT", "1")
with open_router() as router, pytest.raises(StartError, match="OUSAST_START_TIMEOUT.*workspaces not materialised"):
router.start_task("default", "t", "r", {})
assert len(fake_router.requests) >= 2
def test_connection_errors_are_retried_until_the_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OUSAST_ROUTER_URL", "http://127.0.0.1:9")
monkeypatch.setenv("OUSAST_START_TIMEOUT", "1")
with open_router() as router, pytest.raises(StartError, match="within 1s"):
router.start_task("default", "t", "r", {})
def model(secret: SecretKey | None) -> Model:
return Model(metadata=Metadata(name="deepseek-flash"), provider="deepseek", model="deepseek-flash", secret_key=secret)
def test_credentials_come_from_the_models_secret_key_variable() -> None:
key = SecretKey(name="deepseek", key="DEEPSEEK_API_KEY")
assert credentials(None, {}) == {} and credentials(model(None), {"DEEPSEEK_API_KEY": SECRET}) == {}
assert credentials(model(key), {"DEEPSEEK_API_KEY": SECRET}) == {"DEEPSEEK_API_KEY": SECRET}
with pytest.raises(StartError, match="needs DEEPSEEK_API_KEY"):
credentials(model(key), {"DEEPSEEK_API_KEY": ""})
+150 -5
View File
@@ -1,9 +1,12 @@
"""ax runner contract (ai-service-plane task 3, Requirements 4.1-4.4): env contract, readiness, delivery, exit codes."""
"""ax runner contract (ai-service-plane task 3, Requirements 4.1-4.6): env contract, readiness, delivery, exit codes,
the start request and credentials."""
from __future__ import annotations
import hashlib
import io
import json
import logging
import os
import shutil
import signal
@@ -23,9 +26,13 @@ import pytest
from openultrasast.plane import runner
from openultrasast.plane.manifests import GitSource
from openultrasast.plane.router import Router, StartError
from openultrasast.plane.runner import (
EXIT_BY_STATUS,
START_PATH,
DeliveryError,
Start,
StartGate,
clone_destination,
deliver,
find_cached_checkout,
@@ -61,6 +68,7 @@ def env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, stub_root: Path) -> Ite
monkeypatch.delenv(key)
monkeypatch.setenv("AX_RUNNER_HTTP", "0")
monkeypatch.setenv("AX_RUNNER_EXIT_AFTER_COMMAND", "1")
monkeypatch.setenv("AX_RUNNER_AUTOSTART", "1") # the start-request tests below take it away again
monkeypatch.setenv("AX_RUNNER_TASK_PACKAGE", STUB_PACKAGE)
monkeypatch.setenv("OUSAST_STATE_DIR", str(tmp_path / "state"))
monkeypatch.setenv("OUSAST_DELIVERY_BACKOFF", "0")
@@ -341,12 +349,14 @@ def wait_for(predicate: object, timeout: float = 30.0) -> None:
time.sleep(0.05)
def spawn_runner(tmp_path: Path, port: int, **extra: str) -> subprocess.Popen[bytes]:
"""``python -m openultrasast.plane.runner`` as the image runs it: HTTP on, no exit-after-command escape."""
child_env = {k: v for k, v in os.environ.items() if k != "AX_RUNNER_EXIT_AFTER_COMMAND"}
def spawn_runner(tmp_path: Path, port: int, *, autostart: bool = True, log_name: str = "runner.log", **extra: str) -> subprocess.Popen:
"""``python -m openultrasast.plane.runner`` as the image runs it: HTTP on, no exit-after-command escape;
``autostart=False`` also drops the autostart escape, so the command waits for a start request."""
dropped = {"AX_RUNNER_EXIT_AFTER_COMMAND"} | (set() if autostart else {"AX_RUNNER_AUTOSTART"})
child_env = {k: v for k, v in os.environ.items() if k not in dropped}
child_env.update(AX_TASK_YAML=task_yaml(tmp_path / "out", tmp_path / "ws"), AX_WORKSPACES_YAML=files_workspace_yaml())
child_env.update(AX_RUNNER_HTTP="1", AX_RUNNER_PORT=str(port), **extra)
log = (tmp_path / "runner.log").open("wb")
log = (tmp_path / log_name).open("wb")
return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log)
@@ -609,3 +619,138 @@ def test_the_runner_accepts_what_ax_hands_it() -> None:
assert list(spaces) == ["w"]
with pytest.raises(RunnerError, match="uid"):
load_task("apiVersion: ax.io/v1alpha1\nkind: Task\nmetadata:\n name: t\n uid: x\nspec: {}\n")
# --- the start request and credentials (Requirements 4.5, 4.6) -----------------------------------------------------------
SECRET = "sk-test-0f3c9a-never-written-anywhere"
SECRET_SHA = hashlib.sha256(SECRET.encode()).hexdigest()
def post_start(port: int | str, body: object) -> tuple[int, str]:
data = json.dumps(body).encode()
request = urllib.request.Request(f"http://127.0.0.1:{port}{START_PATH}", data=data, method="POST")
try:
with urllib.request.urlopen(request, timeout=5) as response:
return int(response.status), response.read().decode()
except urllib.error.HTTPError as exc:
return exc.code, exc.read().decode()
def ready_on(port: int | str) -> bool:
try:
return get(f"http://127.0.0.1:{port}/readyz") == 200
except urllib.error.URLError:
return False # not listening yet
def files_containing(root: Path, needle: str) -> list[str]:
return sorted(str(p) for p in root.rglob("*") if p.is_file() and needle.encode() in p.read_bytes())
def main_in_thread(tmp_path: Path, **extra: str) -> tuple[threading.Thread, dict[str, str], list[int]]:
"""``runner.main`` on a private environment in a thread: HTTP on a free port, no autostart; the exit code lands
in the returned list once the command was started and delivered."""
environ = {k: v for k, v in os.environ.items() if k != "AX_RUNNER_AUTOSTART"}
environ.update(AX_TASK_YAML=task_yaml(tmp_path / "out", tmp_path / "ws"), AX_WORKSPACES_YAML=files_workspace_yaml())
environ.update(AX_RUNNER_HTTP="1", AX_RUNNER_PORT="0", OUSAST_RUN="run-1", STUB_SECRET_VAR="DEEPSEEK_API_KEY", **extra)
codes: list[int] = []
thread = threading.Thread(target=lambda: codes.append(runner.main(environ=environ)), daemon=True)
thread.start()
wait_for(lambda: "AX_RUNNER_BOUND_PORT" in environ)
wait_for(lambda: ready_on(environ["AX_RUNNER_BOUND_PORT"]))
return thread, environ, codes
def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only_the_child(
env: dict[str, str], receiver: Receiver, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
caplog.set_level(logging.DEBUG)
counter = tmp_path / "runs.txt"
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
port = environ["AX_RUNNER_BOUND_PORT"]
time.sleep(0.5)
assert not counter.exists() and thread.is_alive(), "ready, but the command waits for the start request"
assert post_start(port, {"run": "run-2", "task": "stub"})[0] == 409, "another run's start"
assert post_start(port, {"run": "run-1", "task": "golden"})[0] == 409, "another task's start"
assert post_start(port, {"run": "run-1", "task": "stub", "credentials": {"lower": "x"}})[0] == 400
assert post_start(port, {"run": "run-1", "task": "stub", "credentials": {"OUSAST_OUTPUT_DIR": "/"}})[0] == 400
assert not counter.exists()
status, text = post_start(port, {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}})
assert (status, text) == (202, "started")
status, text = post_start(port, {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}})
assert status == 409 and "already started" in text, "one start per boot"
thread.join(timeout=30)
assert codes == [0] and len(counter.read_text().splitlines()) == 1
facts = json.loads((tmp_path / "out" / "facts.json").read_text())
assert facts["secret_sha256"] == SECRET_SHA, "the credential is in the command's environment"
assert "DEEPSEEK_API_KEY" not in environ and "DEEPSEEK_API_KEY" not in os.environ, "and only there"
assert str(tmp_path / "out" / "facts.json") in files_containing(tmp_path, SECRET_SHA), "the scan reads the output files"
assert files_containing(tmp_path, SECRET) == [], "no file holds the credential"
assert SECRET not in receiver.posts[0][1].decode("latin-1"), "nor the delivered tar"
assert all(SECRET not in record.getMessage() for record in caplog.records), "nor any log record"
assert any("start request: 202" in record.getMessage() for record in caplog.records)
def test_a_crash_that_prints_the_credential_is_redacted(
env: dict[str, str], receiver: Receiver, tmp_path: Path, capfd: pytest.CaptureFixture[str]
) -> None:
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="leak")
start = {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
assert post_start(environ["AX_RUNNER_BOUND_PORT"], start)[0] == 202
thread.join(timeout=30)
assert codes == [EXIT_BY_STATUS["failed"]]
summary = json.loads((tmp_path / "out" / "summary.json").read_text())
assert "auth failed for key [redacted]" in summary["reason"], "the stderr tail keeps the line, not the value"
assert files_containing(tmp_path, SECRET) == []
assert SECRET not in capfd.readouterr().err, "the echoed stderr is redacted too"
def test_start_before_the_workspaces_are_ready_is_503_and_a_delivered_run_refuses_it() -> None:
gate = StartGate()
assert gate.offer(b'{"run": "r", "task": "t"}')[0] == 503, "the reconciler retries until the workspaces are ready"
gate.refuse("task t already ran and was delivered (done)")
status, text = gate.offer(b'{"run": "r", "task": "t"}')
assert status == 409 and "already ran and was delivered" in text
assert "sk-" not in repr(Start({"K": "sk-x"})), "a Start never prints a value"
def test_http_off_without_autostart_is_refused(env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
monkeypatch.delenv("AX_RUNNER_AUTOSTART")
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml())
assert code == EXIT_BY_STATUS["failed"] and not (output / "facts.json").exists()
def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
"""Agent Substrate's golden-snapshot boot runs the same image with the same env; only the router-addressed
actor gets the start request, so only it runs the command. A later boot of a delivered task refuses a start."""
counter = tmp_path / "runs.txt"
same = {"OUSAST_ARTIFACT_URL": receiver.url, "STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1"}
real_port, golden_port = free_port(), free_port()
real = spawn_runner(tmp_path, real_port, autostart=False, log_name="real.log", **same)
golden = spawn_runner(tmp_path, golden_port, autostart=False, log_name="golden.log", **same)
try:
for port in (real_port, golden_port):
wait_for(lambda port=port: ready_on(port))
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET})
marker = tmp_path / "state" / "stub.done.json"
wait_for(marker.exists)
time.sleep(1.0)
assert len(counter.read_text().splitlines()) == 1, "the golden boot never ran the command"
assert golden.poll() is None and len(receiver.posts) == 1
with pytest.raises(StartError, match="already started"):
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {})
again_port = free_port()
again = spawn_runner(tmp_path, again_port, autostart=False, log_name="again.log", **same)
try:
with pytest.raises(StartError, match="already ran and was delivered"):
Router(f"http://127.0.0.1:{again_port}").start_task("default", "stub", "run-1", {})
finally:
again.kill()
again.wait()
finally:
for proc in (real, golden):
proc.kill()
proc.wait()
assert len(counter.read_text().splitlines()) == 1
assert files_containing(tmp_path, SECRET) == [], "no log, workspace, output or state file holds the credential"