mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(plane): router-addressed start signal; credentials only in the start request
Agent Substrate's golden-snapshot boot runs the same image with the same task env, so the runner no longer starts its command by itself: after the workspaces are ready it waits for POST /ousast/v1/start (run/task checked, once per boot, refused after a delivered run). The reconciler sends it through svc/atenet-router with ate-target-actor, which never addresses the golden actor (plane/router.py), and the bound Model's secretKey variable travels only in that body, into the child's env. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ac296ee3a4
commit
14ea48b1ea
@@ -3,11 +3,13 @@
|
||||
The tests copy it into a throwaway package (``AX_RUNNER_TASK_PACKAGE``) so the runner starts it the way it starts
|
||||
a real task, ``python -m <package>.stub_task``. ``STUB_MODE`` picks the outcome (``done``, ``failed``,
|
||||
``unfinished``, ``crash``, ``nosummary``, ``exit``, ``sleep``); ``STUB_COUNTER`` names a file that gets one line
|
||||
per run.
|
||||
per run. ``STUB_SECRET_VAR`` names a credential variable: the stub records the SHA-256 of its value (or None),
|
||||
never the value.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
@@ -23,6 +25,11 @@ def _get(url: str) -> tuple[int, str]:
|
||||
return int(response.status), response.read().decode("utf-8")
|
||||
|
||||
|
||||
def _secret_hash(name: str | None) -> str | None:
|
||||
value = os.environ.get(name) if name else None
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest() if value else None
|
||||
|
||||
|
||||
def main(argv: list[str]) -> None:
|
||||
out = Path(os.environ["OUSAST_OUTPUT_DIR"])
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
@@ -45,6 +52,7 @@ def main(argv: list[str]) -> None:
|
||||
"pgid": os.getpgid(0),
|
||||
"metadata_url": metadata,
|
||||
"metadata_task": _get(f"{metadata}/metadata/v1alpha1/ax/task")[1] if metadata else None,
|
||||
"secret_sha256": _secret_hash(os.environ.get("STUB_SECRET_VAR")),
|
||||
}
|
||||
(out / "facts.json").write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8")
|
||||
if mode == "sleep": # ignore SIGTERM so only the runner's SIGKILL ends the group; the grandchild inherits it
|
||||
@@ -52,6 +60,9 @@ def main(argv: list[str]) -> None:
|
||||
grandchild = subprocess.Popen(["sleep", "60"])
|
||||
(out / "pids.json").write_text(json.dumps([os.getpid(), grandchild.pid]), encoding="utf-8")
|
||||
time.sleep(60)
|
||||
if mode == "leak": # a task that prints its credential and crashes: the runner must redact it
|
||||
sys.stderr.write(f"auth failed for key {os.environ.get(os.environ.get('STUB_SECRET_VAR', ''), '')}\n")
|
||||
raise SystemExit(5)
|
||||
if mode == "crash":
|
||||
raise RuntimeError("stub exploded")
|
||||
if mode == "exit":
|
||||
|
||||
@@ -5,7 +5,8 @@ internal/controller/reconciler.go, internal/server/server.go): ``apply`` creates
|
||||
fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new
|
||||
image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase
|
||||
-- and on the scripted poll it POSTs a scripted tar to the ``OUSAST_ARTIFACT_URL`` in the applied Task's env, or
|
||||
turns ``Failed`` with a condition message, or vanishes. Every outcome comes from a scripted delivery.
|
||||
turns ``Failed`` with a condition message, or vanishes. Every outcome comes from a scripted delivery, and a
|
||||
delivery happens only after the reconciler's start request reached the fake router (Req 4.5).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,7 +17,10 @@ import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -27,6 +31,7 @@ from openultrasast.plane.workspaces import workspaces
|
||||
|
||||
RECONCILER = Path("src/openultrasast/plane/reconciler.py")
|
||||
PIN = "0123456789abcdef0123456789abcdef01234567"
|
||||
SECRET = "sk-reconciler-test-7a2b-only-in-the-start-request"
|
||||
|
||||
# The fields ax's API server decodes (pkg/apis/v1alpha1/ax.proto, protojson: an unknown field is an error at
|
||||
# ``ax apply``). ``None`` marks a value ax types as free-form (a Struct) or that this test does not descend into.
|
||||
@@ -114,6 +119,9 @@ elif argv[:2] == ["get", "task"]:
|
||||
rec.update(phase=script["phase"], message=script.get("message", ""))
|
||||
if script.get("vanish"):
|
||||
rec["gone"] = True
|
||||
started = (HERE / "started" / argv[2]).exists() # the runner runs its command only after the start request
|
||||
if rec["phase"] == "Running" and rec["polls"] >= script.get("polls", 1) and started and not rec.get("sent"):
|
||||
rec["sent"] = True
|
||||
if script.get("deliver", True) and "phase" not in script and not script.get("vanish"):
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w") as tar:
|
||||
@@ -152,6 +160,7 @@ metadata:
|
||||
spec:
|
||||
provider: deepseek
|
||||
model: deepseek-chat
|
||||
secretKey: {name: deepseek, key: DEEPSEEK_API_KEY}
|
||||
parameters: {cache_hit_per_m: 0.07, input_per_m: 0.27, output_per_m: 1.1}
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
@@ -234,6 +243,8 @@ SCRIPT = {
|
||||
|
||||
|
||||
class Fake:
|
||||
router: FakeRouter
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.ax = root / "ax"
|
||||
@@ -263,15 +274,61 @@ class Fake:
|
||||
return out
|
||||
|
||||
|
||||
class FakeRouter(ThreadingHTTPServer):
|
||||
"""Agent Substrate's router as the reconciler sees it: records each start request and answers from ``codes``
|
||||
(by ax task name, default 202); an accepted start marks the task started for the fake ax, whose runner only
|
||||
then runs its command and delivers."""
|
||||
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _RouterHandler)
|
||||
self.root = root
|
||||
self.codes: dict[str, tuple[int, str]] = {}
|
||||
self.starts: list[dict] = []
|
||||
threading.Thread(target=self.serve_forever, daemon=True).start()
|
||||
|
||||
|
||||
class _RouterHandler(BaseHTTPRequestHandler):
|
||||
server: FakeRouter
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)))
|
||||
target = self.headers.get("ate-target-actor", "")
|
||||
name = target.partition("/")[2]
|
||||
code, text = self.server.codes.get(name, (202, "started"))
|
||||
self.server.starts.append({"at": time.monotonic(), "path": self.path, "target": target, "body": body, "code": code})
|
||||
if code == 202:
|
||||
(self.server.root / "started").mkdir(exist_ok=True)
|
||||
(self.server.root / "started" / name).touch()
|
||||
with (self.server.root / "ax.log").open("a") as log:
|
||||
log.write(f"{time.monotonic():.4f} started {name} {code}\n")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Length", str(len(text.encode())))
|
||||
self.end_headers()
|
||||
self.wfile.write(text.encode())
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None: # noqa: A002
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Fake:
|
||||
def fake(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Fake]:
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path / "results"))
|
||||
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
|
||||
monkeypatch.setenv("OUSAST_POLL_SECONDS", "0.02")
|
||||
monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "20") # a test that never delivers fails in seconds, not in 2 hours
|
||||
monkeypatch.setenv("OUSAST_START_TIMEOUT", "2")
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", SECRET) # the Model's secretKey variable, as ``.env`` would set it
|
||||
fake = Fake(tmp_path)
|
||||
fake.script(SCRIPT)
|
||||
return fake
|
||||
fake.router = FakeRouter(tmp_path)
|
||||
monkeypatch.setenv("OUSAST_ROUTER_URL", f"http://127.0.0.1:{fake.router.server_address[1]}")
|
||||
try:
|
||||
yield fake
|
||||
finally:
|
||||
fake.router.shutdown()
|
||||
fake.router.server_close()
|
||||
|
||||
|
||||
def write_run(tmp_path: Path, name: str, tasks: str = CHAIN) -> Path:
|
||||
@@ -426,6 +483,54 @@ def test_no_delivery_within_the_task_timeout_is_failed(fake: Fake, tmp_path: Pat
|
||||
assert [n for _, n in fake.events("delete")] == ["silent-repo-facts"], "ax delete follows every task"
|
||||
|
||||
|
||||
def files_containing(root: Path, needle: str) -> list[str]:
|
||||
return sorted(str(p) for p in root.rglob("*") if p.is_file() and needle.encode() in p.read_bytes())
|
||||
|
||||
|
||||
def test_start_is_sent_once_after_resume_with_the_models_credential(fake: Fake, tmp_path: Path) -> None:
|
||||
"""Req 4.5/4.6: one start per task through the router, after ax accepted a resume; the bound Model's credential
|
||||
travels only in that request, never in the rendered Task, the run dir or any file the run leaves behind."""
|
||||
assert reconciler.run(write_run(tmp_path, "sig"), ax=str(fake.ax)) == "done"
|
||||
starts = fake.router.starts
|
||||
assert [s["target"] for s in starts] == ["default/sig-repo-facts", "default/sig-verify", "default/sig-agree"], "once each"
|
||||
assert all(s["path"] == "/ousast/v1/start" and s["code"] == 202 for s in starts)
|
||||
by_task = {s["body"]["task"]: s for s in starts}
|
||||
assert by_task["sig-verify"]["body"] == {"run": "sig", "task": "sig-verify", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
|
||||
assert by_task["sig-repo-facts"]["body"]["credentials"] == {} and by_task["sig-agree"]["body"]["credentials"] == {}, "no Model"
|
||||
accepted = {}
|
||||
for stamp, words in fake.log(): # the last resume of each task is the one ax accepted
|
||||
if words[:2] == ["resume", "task"]:
|
||||
accepted[words[2]] = stamp
|
||||
delivered = {name: stamp for stamp, name in fake.events("delivered")}
|
||||
for s in starts:
|
||||
name = s["body"]["task"]
|
||||
assert accepted[name] < s["at"] < delivered[name.removeprefix("sig-")], f"{name}: resume, then start, then delivery"
|
||||
verify_model = fake.applied("sig", "verify")["docs"][1]
|
||||
assert verify_model["spec"]["secretKey"] == {"name": "deepseek", "key": "DEEPSEEK_API_KEY"}, "the manifest names the key"
|
||||
assert "DEEPSEEK_API_KEY" not in fake.applied("sig", "verify")["env"], "the rendered Task carries no credential"
|
||||
rendered = str(reconciler.run_dir("sig") / "verify" / "task.yaml")
|
||||
assert rendered in files_containing(tmp_path, "OUSAST_ARTIFACT_URL"), "the scan reads the rendered files it vouches for"
|
||||
assert files_containing(tmp_path, SECRET) == [], "no rendered task.yaml, state, log or applied record holds the value"
|
||||
|
||||
|
||||
def test_a_model_bound_task_without_its_secret_fails_before_ax_sees_it(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("DEEPSEEK_API_KEY")
|
||||
assert reconciler.run(write_run(tmp_path, "nokey"), ax=str(fake.ax)) == "failed"
|
||||
assert statuses("nokey") == {"repo-facts": "done", "verify": "failed", "agree": "pending"}
|
||||
reason = state_of("nokey")["tasks"]["verify"]["reason"]
|
||||
assert "DEEPSEEK_API_KEY" in reason and "deepseek-flash" in reason
|
||||
assert [name for _, name in fake.events("apply")] == ["repo-facts"], "verify is never applied without its credential"
|
||||
|
||||
|
||||
def test_a_refused_start_fails_the_task_with_the_runners_answer(fake: Fake, tmp_path: Path) -> None:
|
||||
fake.router.codes["ref-repo-facts"] = (409, "task ref-repo-facts already ran and was delivered (done)")
|
||||
assert reconciler.run(write_run(tmp_path, "ref"), ax=str(fake.ax)) == "failed"
|
||||
reason = state_of("ref")["tasks"]["repo-facts"]["reason"]
|
||||
assert "HTTP 409" in reason and "already ran and was delivered" in reason
|
||||
assert len(fake.router.starts) == 1, "a refusal is final, not retried"
|
||||
assert [n for _, n in fake.events("delete")] == ["ref-repo-facts"], "ax delete follows a refused start"
|
||||
|
||||
|
||||
def test_outcome_of_needs_a_delivered_summary() -> None:
|
||||
assert reconciler.outcome_of(None) == ("failed", "no summary.json delivered")
|
||||
assert reconciler.outcome_of({"status": "done", "units_done": 1, "units_total": 1}) == ("done", "")
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
"""The start signal through Agent Substrate's router (ai-service-plane Requirements 4.5, 4.6): port-forward
|
||||
lifecycle with a fake kubectl, retry and refusal against a fake router, credentials from the environment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import stat
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.plane.manifests import Metadata, Model, SecretKey
|
||||
from openultrasast.plane.router import START_PATH, StartError, credentials, open_router
|
||||
|
||||
SECRET = "sk-router-test-5d1e-never-logged"
|
||||
|
||||
FAKE_KUBECTL = """#!{python}
|
||||
import json, os, sys, time
|
||||
from pathlib import Path
|
||||
here = Path(__file__).resolve().parent
|
||||
(here / "kubectl.json").write_text(json.dumps({{"argv": sys.argv[1:], "pid": os.getpid()}}))
|
||||
mode = os.environ.get("FAKE_KUBECTL_MODE", "ok")
|
||||
if mode == "fail":
|
||||
print('error: services "atenet-router" not found', flush=True)
|
||||
sys.exit(1)
|
||||
print("Forwarding from 127.0.0.1:{port} -> 80", flush=True)
|
||||
print("Forwarding from [::1]:{port} -> 80", flush=True)
|
||||
time.sleep(600)
|
||||
"""
|
||||
|
||||
|
||||
class FakeRouter(ThreadingHTTPServer):
|
||||
"""Answers the start path from ``codes`` in turn (the last one repeats) and records every request."""
|
||||
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, codes: list[tuple[int, str]]) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _RouterHandler)
|
||||
self.codes = codes
|
||||
self.requests: list[tuple[str, dict[str, str], dict]] = []
|
||||
threading.Thread(target=self.serve_forever, daemon=True).start()
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.server_address[1]}"
|
||||
|
||||
|
||||
class _RouterHandler(BaseHTTPRequestHandler):
|
||||
server: FakeRouter
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)))
|
||||
self.server.requests.append((self.path, {k.lower(): v for k, v in self.headers.items()}, body))
|
||||
code, text = self.server.codes[min(len(self.server.requests), len(self.server.codes)) - 1]
|
||||
payload = text.encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None: # noqa: A002
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_router(request: pytest.FixtureRequest) -> Iterator[FakeRouter]:
|
||||
server = FakeRouter(getattr(request, "param", [(202, "started")]))
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def write_kubectl(tmp_path: Path, port: int) -> Path:
|
||||
path = tmp_path / "kubectl"
|
||||
path.write_text(FAKE_KUBECTL.format(python=sys.executable, port=port), encoding="utf-8")
|
||||
path.chmod(path.stat().st_mode | stat.S_IXUSR)
|
||||
return path
|
||||
|
||||
|
||||
def alive(pid: int) -> bool:
|
||||
try:
|
||||
return Path(f"/proc/{pid}/stat").read_text().split(")")[-1].split()[0] != "Z"
|
||||
except (FileNotFoundError, IndexError):
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for key in ("OUSAST_ROUTER_URL", "OUSAST_START_TIMEOUT", "KIND_CLUSTER_NAME", "FAKE_KUBECTL_MODE"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
|
||||
def test_port_forward_port_is_parsed_and_the_child_is_terminated_on_exit(tmp_path: Path, fake_router: FakeRouter) -> None:
|
||||
port = int(fake_router.server_address[1])
|
||||
kubectl = write_kubectl(tmp_path, port)
|
||||
with open_router(kubectl=str(kubectl)) as router:
|
||||
assert router.url == f"http://127.0.0.1:{port}"
|
||||
seen = json.loads((tmp_path / "kubectl.json").read_text())
|
||||
assert seen["argv"] == ["--context", "kind-ousast", "-n", "ate-system", "port-forward", "svc/atenet-router", ":80"]
|
||||
assert alive(seen["pid"])
|
||||
router.start_task("default", "run-t", "run", {})
|
||||
deadline = time.monotonic() + 5
|
||||
while alive(seen["pid"]) and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
assert not alive(seen["pid"]), "the port-forward child is terminated when the block ends"
|
||||
assert fake_router.requests[0][1]["ate-target-actor"] == "default/run-t"
|
||||
|
||||
|
||||
def test_a_failing_port_forward_raises_with_kubectls_output(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("FAKE_KUBECTL_MODE", "fail")
|
||||
kubectl = write_kubectl(tmp_path, 1)
|
||||
with pytest.raises(StartError, match="atenet-router.*not found"), open_router("kind-other", kubectl=str(kubectl)):
|
||||
pass
|
||||
assert json.loads((tmp_path / "kubectl.json").read_text())["argv"][1] == "kind-other"
|
||||
|
||||
|
||||
def test_router_url_override_starts_no_port_forward(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
|
||||
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
|
||||
with open_router(kubectl=str(tmp_path / "no-such-kubectl")) as router:
|
||||
assert router.url == fake_router.url
|
||||
assert not (tmp_path / "kubectl.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fake_router", [[(503, "no actor yet"), (502, "bad gateway"), (202, "started")]], indirect=True)
|
||||
def test_start_retries_503_and_502_then_succeeds(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
|
||||
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
|
||||
with open_router() as router:
|
||||
router.start_task("default", "chain-verify", "chain", {"DEEPSEEK_API_KEY": SECRET})
|
||||
assert len(fake_router.requests) == 3
|
||||
path, headers, body = fake_router.requests[-1]
|
||||
assert path == START_PATH and headers["ate-target-actor"] == "default/chain-verify"
|
||||
assert body == {"run": "chain", "task": "chain-verify", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fake_router", [[(409, f"this actor runs task 'x' (echo {SECRET})")]], indirect=True)
|
||||
def test_a_409_is_raised_at_once_with_the_response_text_redacted(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
|
||||
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
|
||||
with open_router() as router, pytest.raises(StartError, match="HTTP 409.*runs task 'x'") as caught:
|
||||
router.start_task("default", "t", "r", {"DEEPSEEK_API_KEY": SECRET})
|
||||
assert len(fake_router.requests) == 1 and SECRET not in str(caught.value)
|
||||
assert router.starter("default", "t", "r", {})() is not None, "the starter returns the refusal instead of raising"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fake_router", [[(409, "this task was already started on this boot")]], indirect=True)
|
||||
def test_a_restart_after_re_resume_accepts_a_running_command(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
|
||||
"""First start: 'already started' is a refusal. After a re-resume the actor may not have rebooted; then the
|
||||
runner's 'already started on this boot' means the command still runs, not a failure."""
|
||||
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
|
||||
with open_router() as router:
|
||||
start = router.starter("default", "t", "r", {})
|
||||
assert start() is not None, "a first start that finds the command running is a refusal"
|
||||
assert start() is None, "a re-sent start after a re-resume is not"
|
||||
fake_router.codes = [(409, "task t already ran and was delivered (done)")]
|
||||
assert start() is not None, "any other refusal stays one"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fake_router", [[(503, "workspaces not materialised")]], indirect=True)
|
||||
def test_start_gives_up_after_the_start_timeout(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None:
|
||||
monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url)
|
||||
monkeypatch.setenv("OUSAST_START_TIMEOUT", "1")
|
||||
with open_router() as router, pytest.raises(StartError, match="OUSAST_START_TIMEOUT.*workspaces not materialised"):
|
||||
router.start_task("default", "t", "r", {})
|
||||
assert len(fake_router.requests) >= 2
|
||||
|
||||
|
||||
def test_connection_errors_are_retried_until_the_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OUSAST_ROUTER_URL", "http://127.0.0.1:9")
|
||||
monkeypatch.setenv("OUSAST_START_TIMEOUT", "1")
|
||||
with open_router() as router, pytest.raises(StartError, match="within 1s"):
|
||||
router.start_task("default", "t", "r", {})
|
||||
|
||||
|
||||
def model(secret: SecretKey | None) -> Model:
|
||||
return Model(metadata=Metadata(name="deepseek-flash"), provider="deepseek", model="deepseek-flash", secret_key=secret)
|
||||
|
||||
|
||||
def test_credentials_come_from_the_models_secret_key_variable() -> None:
|
||||
key = SecretKey(name="deepseek", key="DEEPSEEK_API_KEY")
|
||||
assert credentials(None, {}) == {} and credentials(model(None), {"DEEPSEEK_API_KEY": SECRET}) == {}
|
||||
assert credentials(model(key), {"DEEPSEEK_API_KEY": SECRET}) == {"DEEPSEEK_API_KEY": SECRET}
|
||||
with pytest.raises(StartError, match="needs DEEPSEEK_API_KEY"):
|
||||
credentials(model(key), {"DEEPSEEK_API_KEY": ""})
|
||||
+150
-5
@@ -1,9 +1,12 @@
|
||||
"""ax runner contract (ai-service-plane task 3, Requirements 4.1-4.4): env contract, readiness, delivery, exit codes."""
|
||||
"""ax runner contract (ai-service-plane task 3, Requirements 4.1-4.6): env contract, readiness, delivery, exit codes,
|
||||
the start request and credentials."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
@@ -23,9 +26,13 @@ import pytest
|
||||
|
||||
from openultrasast.plane import runner
|
||||
from openultrasast.plane.manifests import GitSource
|
||||
from openultrasast.plane.router import Router, StartError
|
||||
from openultrasast.plane.runner import (
|
||||
EXIT_BY_STATUS,
|
||||
START_PATH,
|
||||
DeliveryError,
|
||||
Start,
|
||||
StartGate,
|
||||
clone_destination,
|
||||
deliver,
|
||||
find_cached_checkout,
|
||||
@@ -61,6 +68,7 @@ def env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, stub_root: Path) -> Ite
|
||||
monkeypatch.delenv(key)
|
||||
monkeypatch.setenv("AX_RUNNER_HTTP", "0")
|
||||
monkeypatch.setenv("AX_RUNNER_EXIT_AFTER_COMMAND", "1")
|
||||
monkeypatch.setenv("AX_RUNNER_AUTOSTART", "1") # the start-request tests below take it away again
|
||||
monkeypatch.setenv("AX_RUNNER_TASK_PACKAGE", STUB_PACKAGE)
|
||||
monkeypatch.setenv("OUSAST_STATE_DIR", str(tmp_path / "state"))
|
||||
monkeypatch.setenv("OUSAST_DELIVERY_BACKOFF", "0")
|
||||
@@ -341,12 +349,14 @@ def wait_for(predicate: object, timeout: float = 30.0) -> None:
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def spawn_runner(tmp_path: Path, port: int, **extra: str) -> subprocess.Popen[bytes]:
|
||||
"""``python -m openultrasast.plane.runner`` as the image runs it: HTTP on, no exit-after-command escape."""
|
||||
child_env = {k: v for k, v in os.environ.items() if k != "AX_RUNNER_EXIT_AFTER_COMMAND"}
|
||||
def spawn_runner(tmp_path: Path, port: int, *, autostart: bool = True, log_name: str = "runner.log", **extra: str) -> subprocess.Popen:
|
||||
"""``python -m openultrasast.plane.runner`` as the image runs it: HTTP on, no exit-after-command escape;
|
||||
``autostart=False`` also drops the autostart escape, so the command waits for a start request."""
|
||||
dropped = {"AX_RUNNER_EXIT_AFTER_COMMAND"} | (set() if autostart else {"AX_RUNNER_AUTOSTART"})
|
||||
child_env = {k: v for k, v in os.environ.items() if k not in dropped}
|
||||
child_env.update(AX_TASK_YAML=task_yaml(tmp_path / "out", tmp_path / "ws"), AX_WORKSPACES_YAML=files_workspace_yaml())
|
||||
child_env.update(AX_RUNNER_HTTP="1", AX_RUNNER_PORT=str(port), **extra)
|
||||
log = (tmp_path / "runner.log").open("wb")
|
||||
log = (tmp_path / log_name).open("wb")
|
||||
return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log)
|
||||
|
||||
|
||||
@@ -609,3 +619,138 @@ def test_the_runner_accepts_what_ax_hands_it() -> None:
|
||||
assert list(spaces) == ["w"]
|
||||
with pytest.raises(RunnerError, match="uid"):
|
||||
load_task("apiVersion: ax.io/v1alpha1\nkind: Task\nmetadata:\n name: t\n uid: x\nspec: {}\n")
|
||||
|
||||
|
||||
# --- the start request and credentials (Requirements 4.5, 4.6) -----------------------------------------------------------
|
||||
|
||||
SECRET = "sk-test-0f3c9a-never-written-anywhere"
|
||||
SECRET_SHA = hashlib.sha256(SECRET.encode()).hexdigest()
|
||||
|
||||
|
||||
def post_start(port: int | str, body: object) -> tuple[int, str]:
|
||||
data = json.dumps(body).encode()
|
||||
request = urllib.request.Request(f"http://127.0.0.1:{port}{START_PATH}", data=data, method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return int(response.status), response.read().decode()
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read().decode()
|
||||
|
||||
|
||||
def ready_on(port: int | str) -> bool:
|
||||
try:
|
||||
return get(f"http://127.0.0.1:{port}/readyz") == 200
|
||||
except urllib.error.URLError:
|
||||
return False # not listening yet
|
||||
|
||||
|
||||
def files_containing(root: Path, needle: str) -> list[str]:
|
||||
return sorted(str(p) for p in root.rglob("*") if p.is_file() and needle.encode() in p.read_bytes())
|
||||
|
||||
|
||||
def main_in_thread(tmp_path: Path, **extra: str) -> tuple[threading.Thread, dict[str, str], list[int]]:
|
||||
"""``runner.main`` on a private environment in a thread: HTTP on a free port, no autostart; the exit code lands
|
||||
in the returned list once the command was started and delivered."""
|
||||
environ = {k: v for k, v in os.environ.items() if k != "AX_RUNNER_AUTOSTART"}
|
||||
environ.update(AX_TASK_YAML=task_yaml(tmp_path / "out", tmp_path / "ws"), AX_WORKSPACES_YAML=files_workspace_yaml())
|
||||
environ.update(AX_RUNNER_HTTP="1", AX_RUNNER_PORT="0", OUSAST_RUN="run-1", STUB_SECRET_VAR="DEEPSEEK_API_KEY", **extra)
|
||||
codes: list[int] = []
|
||||
thread = threading.Thread(target=lambda: codes.append(runner.main(environ=environ)), daemon=True)
|
||||
thread.start()
|
||||
wait_for(lambda: "AX_RUNNER_BOUND_PORT" in environ)
|
||||
wait_for(lambda: ready_on(environ["AX_RUNNER_BOUND_PORT"]))
|
||||
return thread, environ, codes
|
||||
|
||||
|
||||
def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only_the_child(
|
||||
env: dict[str, str], receiver: Receiver, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
caplog.set_level(logging.DEBUG)
|
||||
counter = tmp_path / "runs.txt"
|
||||
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
|
||||
port = environ["AX_RUNNER_BOUND_PORT"]
|
||||
time.sleep(0.5)
|
||||
assert not counter.exists() and thread.is_alive(), "ready, but the command waits for the start request"
|
||||
assert post_start(port, {"run": "run-2", "task": "stub"})[0] == 409, "another run's start"
|
||||
assert post_start(port, {"run": "run-1", "task": "golden"})[0] == 409, "another task's start"
|
||||
assert post_start(port, {"run": "run-1", "task": "stub", "credentials": {"lower": "x"}})[0] == 400
|
||||
assert post_start(port, {"run": "run-1", "task": "stub", "credentials": {"OUSAST_OUTPUT_DIR": "/"}})[0] == 400
|
||||
assert not counter.exists()
|
||||
status, text = post_start(port, {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}})
|
||||
assert (status, text) == (202, "started")
|
||||
status, text = post_start(port, {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}})
|
||||
assert status == 409 and "already started" in text, "one start per boot"
|
||||
thread.join(timeout=30)
|
||||
assert codes == [0] and len(counter.read_text().splitlines()) == 1
|
||||
facts = json.loads((tmp_path / "out" / "facts.json").read_text())
|
||||
assert facts["secret_sha256"] == SECRET_SHA, "the credential is in the command's environment"
|
||||
assert "DEEPSEEK_API_KEY" not in environ and "DEEPSEEK_API_KEY" not in os.environ, "and only there"
|
||||
assert str(tmp_path / "out" / "facts.json") in files_containing(tmp_path, SECRET_SHA), "the scan reads the output files"
|
||||
assert files_containing(tmp_path, SECRET) == [], "no file holds the credential"
|
||||
assert SECRET not in receiver.posts[0][1].decode("latin-1"), "nor the delivered tar"
|
||||
assert all(SECRET not in record.getMessage() for record in caplog.records), "nor any log record"
|
||||
assert any("start request: 202" in record.getMessage() for record in caplog.records)
|
||||
|
||||
|
||||
def test_a_crash_that_prints_the_credential_is_redacted(
|
||||
env: dict[str, str], receiver: Receiver, tmp_path: Path, capfd: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="leak")
|
||||
start = {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}}
|
||||
assert post_start(environ["AX_RUNNER_BOUND_PORT"], start)[0] == 202
|
||||
thread.join(timeout=30)
|
||||
assert codes == [EXIT_BY_STATUS["failed"]]
|
||||
summary = json.loads((tmp_path / "out" / "summary.json").read_text())
|
||||
assert "auth failed for key [redacted]" in summary["reason"], "the stderr tail keeps the line, not the value"
|
||||
assert files_containing(tmp_path, SECRET) == []
|
||||
assert SECRET not in capfd.readouterr().err, "the echoed stderr is redacted too"
|
||||
|
||||
|
||||
def test_start_before_the_workspaces_are_ready_is_503_and_a_delivered_run_refuses_it() -> None:
|
||||
gate = StartGate()
|
||||
assert gate.offer(b'{"run": "r", "task": "t"}')[0] == 503, "the reconciler retries until the workspaces are ready"
|
||||
gate.refuse("task t already ran and was delivered (done)")
|
||||
status, text = gate.offer(b'{"run": "r", "task": "t"}')
|
||||
assert status == 409 and "already ran and was delivered" in text
|
||||
assert "sk-" not in repr(Start({"K": "sk-x"})), "a Start never prints a value"
|
||||
|
||||
|
||||
def test_http_off_without_autostart_is_refused(env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
monkeypatch.delenv("AX_RUNNER_AUTOSTART")
|
||||
code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml())
|
||||
assert code == EXIT_BY_STATUS["failed"] and not (output / "facts.json").exists()
|
||||
|
||||
|
||||
def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
|
||||
"""Agent Substrate's golden-snapshot boot runs the same image with the same env; only the router-addressed
|
||||
actor gets the start request, so only it runs the command. A later boot of a delivered task refuses a start."""
|
||||
counter = tmp_path / "runs.txt"
|
||||
same = {"OUSAST_ARTIFACT_URL": receiver.url, "STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1"}
|
||||
real_port, golden_port = free_port(), free_port()
|
||||
real = spawn_runner(tmp_path, real_port, autostart=False, log_name="real.log", **same)
|
||||
golden = spawn_runner(tmp_path, golden_port, autostart=False, log_name="golden.log", **same)
|
||||
try:
|
||||
for port in (real_port, golden_port):
|
||||
wait_for(lambda port=port: ready_on(port))
|
||||
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET})
|
||||
marker = tmp_path / "state" / "stub.done.json"
|
||||
wait_for(marker.exists)
|
||||
time.sleep(1.0)
|
||||
assert len(counter.read_text().splitlines()) == 1, "the golden boot never ran the command"
|
||||
assert golden.poll() is None and len(receiver.posts) == 1
|
||||
with pytest.raises(StartError, match="already started"):
|
||||
Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {})
|
||||
again_port = free_port()
|
||||
again = spawn_runner(tmp_path, again_port, autostart=False, log_name="again.log", **same)
|
||||
try:
|
||||
with pytest.raises(StartError, match="already ran and was delivered"):
|
||||
Router(f"http://127.0.0.1:{again_port}").start_task("default", "stub", "run-1", {})
|
||||
finally:
|
||||
again.kill()
|
||||
again.wait()
|
||||
finally:
|
||||
for proc in (real, golden):
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
assert len(counter.read_text().splitlines()) == 1
|
||||
assert files_containing(tmp_path, SECRET) == [], "no log, workspace, output or state file holds the credential"
|
||||
|
||||
Reference in New Issue
Block a user