From 14ea48b1eaa6b4e59b35cc183e2f93a49ff49cff Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:41:35 +0200 Subject: [PATCH] feat(plane): router-addressed start signal; credentials only in the start request Agent Substrate's golden-snapshot boot runs the same image with the same task env, so the runner no longer starts its command by itself: after the workspaces are ready it waits for POST /ousast/v1/start (run/task checked, once per boot, refused after a delivered run). The reconciler sends it through svc/atenet-router with ate-target-actor, which never addresses the golden actor (plane/router.py), and the bound Model's secretKey variable travels only in that body, into the child's env. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .kiro/specs/ai-service-plane/design.md | 33 +++ .kiro/specs/ai-service-plane/tasks.md | 29 ++- .../measurements/2026-09-08-module-audit.json | 15 +- src/openultrasast/plane/reconciler.py | 122 +++++------ src/openultrasast/plane/router.py | 170 ++++++++++++++++ src/openultrasast/plane/runner.py | 177 ++++++++++++++-- tests/plane_stub_task.py | 13 +- tests/test_plane_reconciler.py | 111 +++++++++- tests/test_plane_router.py | 189 ++++++++++++++++++ tests/test_plane_runner.py | 155 +++++++++++++- 10 files changed, 919 insertions(+), 95 deletions(-) create mode 100644 src/openultrasast/plane/router.py create mode 100644 tests/test_plane_router.py diff --git a/.kiro/specs/ai-service-plane/design.md b/.kiro/specs/ai-service-plane/design.md index 1c2f4f5..3e09905 100644 --- a/.kiro/specs/ai-service-plane/design.md +++ b/.kiro/specs/ai-service-plane/design.md @@ -76,6 +76,27 @@ with a failed delivery retries the delivery only. SIGTERM (ax's stop and suspend process group, followed by SIGKILL after `OUSAST_TERM_GRACE` (10 s), and the runner exits 0. `AX_RUNNER_EXIT_AFTER_COMMAND=1` is a unit-test escape that exits with the task's code after delivery. +The command never starts by itself (Requirements 4.5, 4.6). Once the workspaces are ready the runner serves +`POST /ousast/v1/start` on port 80 with the body `{"run", "task", "credentials"}`: 503 before the workspaces are +ready (the reconciler retries), 409 when `run`/`task` differ from `OUSAST_RUN` and the Task's name, 409 for a +second request on the same boot, 409 with the reason when the completion marker records a delivered run, 400 for +a malformed body or a credential name outside `^[A-Z][A-Z0-9_]*$` (or under `AX_`/`OUSAST_`), and 202 once. +Only then does the command run -- or, when the marker records a failed delivery, only the delivery is retried; a +runner failure (bad manifests, a workspace that cannot be materialised) is likewise delivered only after a start, +or refused with its reason. The credentials stay in the runner's memory and are merged into the child's +environment only; they are never logged or written, and their values are redacted from the child's echoed +stderr and the stderr tail a crash summary keeps. `AX_RUNNER_AUTOSTART=1` (unit tests only; required with +`AX_RUNNER_HTTP=0`) starts without a request. + +**Golden snapshot.** Agent Substrate boots every new ActorTemplate once as a temporary "golden" actor (atespace +`ate-golden`, a UUID name) to capture a snapshot. ax bakes `AX_TASK_YAML`, `AX_WORKSPACES_YAML` and `spec.env` +into the template, and inside the sandbox the hostname is always `actor` with no identity variable, so the golden +boot and the real one are indistinguishable from within (observed on the live cluster, 2026-09-29: the command +ran in both). The distinction exists only outside: Substrate's router (`svc/atenet-router` in `ate-system`) +routes on `ate-target-actor: /` to the task's actor, resuming it if suspended, and never to the +golden one. Hence the start request, and hence credentials only in it: a template, a rendered Task or an ax +manifest is visible to the golden boot and to anyone who can read the cluster's objects. + ### Reconciler (`reconciler.py`, ax-backed) `ousast plane run `, `ousast plane status ` and `ousast plane doctor`. State: `state.json` with @@ -98,6 +119,18 @@ the token attribution table (Requirement 7.3) from the delivered summaries and w the others re-executed; per-unit resume is the task's own duty (Requirement 2.2). No prompts, scoring or model calls anywhere in this module; a test counts its lines. +Start (Requirements 4.5, 4.6; `router.py`, kept out of the reconciler's line budget): for the whole run the +reconciler holds one `kubectl --context kind-ousast -n ate-system port-forward svc/atenet-router :80` (random local +port read from kubectl's `Forwarding from 127.0.0.1:NNNN`; the child is terminated when the run ends; +`OUSAST_ROUTER_URL` replaces it, as in the tests). When ax reports `Running` after an accepted resume -- again +after a re-resume of a suspended task, where an "already started on this boot" answer means the command +survived and still runs -- it posts the start body with `ate-target-actor: /`, +retrying 502/503/504 and connection errors with backoff up to `OUSAST_START_TIMEOUT` (300 s); a 409 or the timeout +fails the task with the response text unless it already delivered. Credentials come from the Model binding: the +bound Model's `spec.secretKey.key` names the variable (`DEEPSEEK_API_KEY` for `deepseek-flash`), read from the +reconciler's own environment (`ousast` loads `.env` at startup); a Model-bound task whose variable is unset fails +before `ax apply`, and a task without a Model gets none. The value is never rendered, logged or written. + ### Budget (`budget.py`) Wraps a chat client: sums the provider's usage fields per call (`prompt_tokens`, `prompt_cache_hit_tokens`, diff --git a/.kiro/specs/ai-service-plane/tasks.md b/.kiro/specs/ai-service-plane/tasks.md index 2a31e93..3c35656 100644 --- a/.kiro/specs/ai-service-plane/tasks.md +++ b/.kiro/specs/ai-service-plane/tasks.md @@ -41,12 +41,20 @@ mypy are clean, and the commit is gated on pytest's own exit code. Script-level write `/workspace/.ousast-state/.done.json`, and keep serving until SIGTERM (ax's contract for PID 1); a boot with the marker never reruns the command and retries only a failed delivery; SIGTERM goes to the command's group, SIGKILL after `OUSAST_TERM_GRACE`, exit 0. `AX_RUNNER_EXIT_AFTER_COMMAND=1` (tests only) - exits after delivery with 0 done, 2 failed, 3 unfinished. `plane/Dockerfile.runner`: this package on Python 3.12, installed to + exits after delivery with 0 done, 2 failed, 3 unfinished. The command never starts by itself: after the + workspaces are ready the runner waits for `POST /ousast/v1/start` (`{run, task, credentials}`; 503 before + ready, 409 on a run/task mismatch, a second request, or a marker recording a delivered run; 202 once), + merges the credentials into the child's environment only and redacts them from its echoed stderr, so + Agent Substrate's golden-snapshot boot runs nothing (Req 4.5, 4.6). `AX_RUNNER_AUTOSTART=1` (tests only) + starts without a request. `plane/Dockerfile.runner`: this package on Python 3.12, installed to `/usr/local/bin/ax-task-runner`, loaded into kind by `ops/ax/up.sh`. - `tests/test_plane_runner.py`: the env contract on a fixture workspace; readyz 503 then 200; delivery to a local receiver; delivery failure yields `failed`; the child's cwd and `AX_METADATA_URL`; the runner stays - up after the command; SIGTERM kills a long-running command's group; the marker prevents a rerun. - - _Requirements: 4.1, 4.2, 4.3, 4.4_ + up after the command; SIGTERM kills a long-running command's group; the marker prevents a rerun; no command + before a start request, 409 on a wrong run/task and on a second start, the credential reaches the child + (its hash in the artifact) and no file or log record, a crash printing it is redacted, and of two runners + with identical env only the addressed one runs (the golden scenario). + - _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5, 4.6_ - [ ] 4. ax-backed reconciler and CLI - `src/openultrasast/plane/reconciler.py` (under 500 lines, no pipeline logic, no local subprocess @@ -56,7 +64,12 @@ mypy are clean, and the commit is gated on pytest's own exit code. Script-level none for a finished command); `Failed` or a vanished task before delivery, or no delivery within `OUSAST_TASK_TIMEOUT`, fails the task with ax's condition message; the artifact HTTP receiver, `serialize` labels never overlapping, `--workers`, budgets and artifact env passed in the rendered Task manifest, outcome read from the delivered `summary.json`, skip - `done` on rerun, PID lock per run, `ax delete` of finished Tasks. + `done` on rerun, PID lock per run, `ax delete` of finished Tasks. Once ax reports `Running` after a + resume, one start request per resume through Agent Substrate's router (`src/openultrasast/plane/router.py`: + `kubectl port-forward svc/atenet-router` for the run, or `OUSAST_ROUTER_URL`; header + `ate-target-actor: /`; 502/503/504 and connection errors retried up to + `OUSAST_START_TIMEOUT`, a 409 final), carrying the bound Model's `secretKey.key` variable from the + reconciler's environment; a Model-bound task whose variable is unset fails before `ax apply`. - `ousast plane run `, `ousast plane status ` (token attribution table per task and Model from the tasks' `summary.json` usage; `--units` from `units.jsonl`; also written to `attribution.json`), `ousast plane workspaces ` (generates Workspace manifests per case pin). @@ -64,9 +77,13 @@ mypy are clean, and the commit is gated on pytest's own exit code. Script-level failing DeadlineExceeded, Running without a Completed phase) drives a DAG: order, non-overlap, skip-on-rerun, `unfinished` and `failed` propagation, resume retry and timeout, `Failed` with a condition message, the task timeout without delivery, the lock, the attribution table, and the - line-count assertion. `tests/test_plane_ax_live.py` (marker `ax`, skipped without a live cluster): one + line-count assertion; with a fake router: one start per task after the accepted resume and before the + delivery, the credential in the verify start only and in no file under the run or tmp dirs, a missing + secret failing the task, a refused start failing it. `tests/test_plane_router.py`: port parsing from a + fake kubectl and its termination on exit, retry on 503/502 then 202, 409 raised with redacted text, the + start timeout. `tests/test_plane_ax_live.py` (marker `ax`, skipped without a live cluster): one trivial Task end to end. - - _Requirements: 3.1, 3.2, 3.3, 3.4, 2.2, 7.1, 7.3_ + - _Requirements: 3.1, 3.2, 3.3, 3.4, 2.2, 4.5, 4.6, 7.1, 7.3_ - [ ] 5. `repo-facts` task - `src/openultrasast/plane/tasks/repo_facts.py`: product files, functions per file (per-language diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 0bdcfd6..8f2cc7c 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,7 +1,7 @@ { - "modules": 130, + "modules": 131, "counts": { - "load_bearing": 121, + "load_bearing": 122, "standalone_capability": 9 }, "orphaned": [], @@ -661,6 +661,7 @@ "cli", "plane.generate", "plane.reconciler", + "plane.router", "plane.runner", "plane.tasks.verify", "plane.workspaces" @@ -697,6 +698,7 @@ "importers": [ "plane.generate", "plane.reconciler", + "plane.router", "plane.runner", "plane.workspaces" ] @@ -710,6 +712,15 @@ "plane.workspaces" ] }, + { + "module": "plane.router", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "plane.reconciler", + "plane.runner" + ] + }, { "module": "plane.runner", "classification": "load_bearing", diff --git a/src/openultrasast/plane/reconciler.py b/src/openultrasast/plane/reconciler.py index cee037d..bfd23d3 100644 --- a/src/openultrasast/plane/reconciler.py +++ b/src/openultrasast/plane/reconciler.py @@ -1,26 +1,25 @@ -"""ax-backed reconciler of a ``Run``: submission, resume, completion by delivery (ai-service-plane Req 3, 4.4, 7). +"""ax-backed reconciler of a ``Run``: submission, resume, start, completion by delivery (ai-service-plane Req 3, 4, 7). -Every task of a Run is executed by ax on this host's kind cluster; nothing here runs a task as a local subprocess -and nothing here knows a prompt, a score or a model call. For each ready task the reconciler renders one YAML -file (the Task with its env extended, its Workspaces, one generated Workspace carrying the consumed artifacts as -``files`` entries, and the bound Model) and runs ``ax apply -f``. ax creates a Task ``Suspended``; the reconciler -resumes it (``ax resume task``), retrying with backoff while ax answers DeadlineExceeded/Unavailable (the first -resume of a new image waits for its golden snapshot) for up to ``OUSAST_RESUME_TIMEOUT`` s (default 900). ax has no -Completed phase and never reports that a command exited; completion is the runner's artifact delivery, and -``Failed``, the task vanishing, or ``OUSAST_TASK_TIMEOUT`` s (default 7200) without delivery fail the task with -ax's condition message. ``ax delete task`` follows either way. The outcome is the delivered ``summary.json``. +ax executes every task on this host's kind cluster; nothing here runs a task locally or knows a prompt, a score or +a model call. Per ready task: one rendered YAML file (the Task with its env extended, its Workspaces, a generated +Workspace with the consumed artifacts as ``files``, the bound Model), ``ax apply -f``, ``ax resume task`` (retried +on DeadlineExceeded/Unavailable -- a new image's first resume waits for Agent Substrate's golden snapshot -- for +up to ``OUSAST_RESUME_TIMEOUT`` s, 900), and once ax says ``Running`` one start request through Substrate's router +(``router.py``, Req 4.5) carrying the bound Model's credential from this process's environment, never from a file +(Req 4.6). Completion is the runner's artifact delivery (ax has no Completed phase); ``Failed``, the task +vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without delivery fail it; ``ax delete task`` follows. -ax's API server rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/*`` -annotations reach the task as env: ``openultrasast.io/model`` as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req -7.1), each Workspace's ``openultrasast.io/git-commits`` as ``OUSAST_GIT_PINS``. ``OUSAST_INPUT_`` points into -a generated Workspace at ``/workspace/.ousast-in/``. The receiver takes the runner's tar (``X-Ousast-Run``, -``X-Ousast-Task``) on ``OUSAST_ARTIFACT_PORT`` and advertises ``OUSAST_ARTIFACT_HOST`` (default ``172.17.0.1``, the -docker bridge kind nodes reach). Run state: ``~/ousast-results/plane//`` (``OUSAST_RESULTS``), polls every -``OUSAST_POLL_SECONDS`` (3). +ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the +task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's ``openultrasast.io/git-commits`` +as ``OUSAST_GIT_PINS``; ``OUSAST_INPUT_`` points into ``/workspace/.ousast-in/``. The receiver takes +the runner's tar (``X-Ousast-Run``, ``X-Ousast-Task``) on ``OUSAST_ARTIFACT_PORT``, advertised as +``OUSAST_ARTIFACT_HOST`` (default ``172.17.0.1``, the docker bridge). State: ``~/ousast-results/plane//`` +(``OUSAST_RESULTS``), polled every ``OUSAST_POLL_SECONDS`` (3). """ from __future__ import annotations +import contextlib import io import json import os @@ -41,6 +40,7 @@ import yaml from .doctor import doctor from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests +from .router import Router, credentials, open_router MODEL_ANNOTATION = "openultrasast.io/model" OUTPUT_ROOT = "/workspace/.ousast-out" @@ -297,27 +297,22 @@ def _acquire_lock(base: Path) -> Path: lock = base / "lock" pid = _read_json(lock) if isinstance(pid, int) and pid != os.getpid(): - try: + with contextlib.suppress(OSError): os.kill(pid, 0) - except OSError: - pass - else: raise RuntimeError(f"run is held by pid {pid} ({lock})") lock.write_text(json.dumps(os.getpid()), encoding="utf-8") return lock -def _await(ax: Ax, name: str, delivered: Callable[[], bool], poll: float) -> str | None: - """Resume ``name`` and wait for its delivery: None once delivered, else why the task failed. - - ax sets ``Failed`` when a resume call fails, so ``Failed`` is final only once a resume went through; until then - a DeadlineExceeded/Unavailable resume is retried with doubling backoff inside ``OUSAST_RESUME_TIMEOUT``. - """ - task_limit = float(os.environ.get("OUSAST_TASK_TIMEOUT") or 7200) +def _await(ax: Ax, name: str, delivered: Callable[[], bool], start: Callable[[], str | None]) -> str | None: + """Resume ``name``, ``start`` it once Running (again after a re-resume), await delivery: None, else why it failed. + ``Failed`` is final only once a resume went through (ax sets it when a resume call fails); until then a + DeadlineExceeded/Unavailable resume is retried with doubling backoff inside ``OUSAST_RESUME_TIMEOUT``.""" + task_limit, poll = float(os.environ.get("OUSAST_TASK_TIMEOUT") or 7200), float(os.environ.get("OUSAST_POLL_SECONDS") or 3) resume_limit = float(os.environ.get("OUSAST_RESUME_TIMEOUT") or 900) started = time.monotonic() resuming: float | None = started # when the current resume spell began; None while ax runs the task - retry_at, backoff, last, unknown, phase = started, poll, "", 0, "Suspended" + retry_at, backoff, last, unknown, phase, sent = started, poll, "", 0, "Suspended", False while not delivered(): now = time.monotonic() if now - started > task_limit: @@ -344,22 +339,28 @@ def _await(ax: Ax, name: str, delivered: Callable[[], bool], poll: float) -> str if resuming is None and phase == "Failed": return f"ax phase Failed: {message}" if resuming is None and phase == "Suspended": - resuming, retry_at, backoff = time.monotonic(), 0.0, poll + resuming, retry_at, backoff, sent = time.monotonic(), 0.0, poll, False + if resuming is None and phase == "Running" and not sent: + sent, refused = True, start() # through Substrate's router: the golden actor is never addressed (Req 4.5) + if refused and not delivered(): + return refused return None -def _execute(ax: Ax, run: Run, entry: RunTask, manifests: Manifests, base: Path, receiver: Receiver, poll: float) -> tuple[str, str]: - ax_name = _ax_name(run.metadata.name, entry.name) +def _execute(ax: Ax, run: Run, entry: RunTask, manifests: Manifests, base: Path, receiver: Receiver, router: Router) -> tuple[str, str]: + ax_name, task = _ax_name(run.metadata.name, entry.name), manifests.tasks[entry.task] manifest = base / entry.name / "task.yaml" manifest.parent.mkdir(parents=True, exist_ok=True) docs = render_task(run, entry, manifests, base, receiver.url()) manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8") - try: + try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request + secrets = credentials(manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "")) + start = router.starter(task.metadata.atespace or "default", ax_name, run.metadata.name, secrets) ax.apply(manifest) - except RuntimeError as exc: + except RuntimeError as exc: # StartError included: a Model-bound task whose secret is unset return "failed", str(exc) try: - failure = _await(ax, ax_name, lambda: entry.name in receiver.delivered, poll) + failure = _await(ax, ax_name, lambda: entry.name in receiver.delivered, start) finally: ax.delete(ax_name) return ("failed", failure) if failure else outcome_of(_read_json(base / entry.name / "summary.json")) @@ -371,9 +372,7 @@ def run(run_manifest: Path, *, workers: int = 1, ax: str = "ax", results_root: P base = run_dir(run_spec.metadata.name, results_root) base.mkdir(parents=True, exist_ok=True) lock = _acquire_lock(base) - poll = float(os.environ.get("OUSAST_POLL_SECONDS") or 3) - fresh = {"run": run_spec.metadata.name, "started": _now(), "tasks": {}} - state = _State(base / "state.json", _read_json(base / "state.json") or fresh) + state = _State(base / "state.json", _read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}}) receiver = Receiver(run_spec.metadata.name, base, int(os.environ.get("OUSAST_ARTIFACT_PORT") or 0)) threading.Thread(target=receiver.serve_forever, daemon=True).start() cli, finished = Ax(ax), queue.Queue[tuple[str, str, str]]() @@ -387,31 +386,32 @@ def run(run_manifest: Path, *, workers: int = 1, ax: str = "ax", results_root: P if state.status(entry.name) in ("running", "suspended"): cli.delete(_ax_name(run_spec.metadata.name, entry.name)) # an interrupted attempt; ax must not keep it state.set(entry.name, status="running", started=_now(), finished=None) - finished.put((entry.name, *_execute(cli, run_spec, entry, manifests, base, receiver, poll))) + finished.put((entry.name, *_execute(cli, run_spec, entry, manifests, base, receiver, router))) try: - while pending or running: - blocked = {t.serialize for t in running.values() if t.serialize} - for entry in list(pending): - if halted or len(running) >= max(workers, 1) or (entry.serialize and entry.serialize in blocked): - continue - if not all(state.status(p) == "done" for p in entry.producers): - continue - pending.remove(entry) - running[entry.name] = entry - blocked.add(entry.serialize or "") - with receiver.lock: - receiver.running.add(entry.name) - threading.Thread(target=worker, args=(entry,), daemon=True).start() - if not running: - break # nothing can start: a producer stopped short - name, status, reason = finished.get() - del running[name] - summary = _read_json(base / name / "summary.json") or {} - state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None) - halted = halted or status == "failed" - if status == "unfinished": - pending = [t for t in pending if name not in t.producers] + with open_router() as router: # one port-forward to Substrate's router for the whole run + while pending or running: + blocked = {t.serialize for t in running.values() if t.serialize} + for entry in list(pending): + if halted or len(running) >= max(workers, 1) or (entry.serialize and entry.serialize in blocked): + continue + if not all(state.status(p) == "done" for p in entry.producers): + continue + pending.remove(entry) + running[entry.name] = entry + blocked.add(entry.serialize or "") + with receiver.lock: + receiver.running.add(entry.name) + threading.Thread(target=worker, args=(entry,), daemon=True).start() + if not running: + break # nothing can start: a producer stopped short + name, status, reason = finished.get() + del running[name] + summary = _read_json(base / name / "summary.json") or {} + state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None) + halted = halted or status == "failed" + if status == "unfinished": + pending = [t for t in pending if name not in t.producers] finally: receiver.shutdown() lock.unlink(missing_ok=True) diff --git a/src/openultrasast/plane/router.py b/src/openultrasast/plane/router.py new file mode 100644 index 0000000..80c6503 --- /dev/null +++ b/src/openultrasast/plane/router.py @@ -0,0 +1,170 @@ +"""The start signal through Agent Substrate's router (ai-service-plane Requirements 4.5, 4.6). + +Agent Substrate boots every new ActorTemplate once as a temporary "golden" actor to capture a snapshot, with the +same image and the same task environment ax baked into the template, so a runner cannot tell that boot from the +real one. The runner therefore never starts its command by itself: it prepares the workspaces, reports ready and +waits for ``POST /ousast/v1/start``. The reconciler sends that request through Substrate's router +(``svc/atenet-router`` in ``ate-system``), which routes on ``ate-target-actor: /`` to the task's +actor only -- never to the golden one -- and resumes a suspended actor before proxying. + +:func:`open_router` port-forwards the router to a random local port for the duration of a Run (or uses +``OUSAST_ROUTER_URL`` when set, as the tests do); :meth:`Router.start_task` posts the start body, retrying 502/503/504 +and connection errors with backoff for up to ``OUSAST_START_TIMEOUT`` s (default 300). Provider credentials travel +only in that body: :func:`credentials` reads the variable a bound Model's ``secretKey`` names from this process's +environment (``ousast`` loads ``.env`` at startup); nothing here writes or logs a value. +""" + +from __future__ import annotations + +import contextlib +import http.client +import json +import os +import queue +import re +import subprocess +import threading +import time +import urllib.error +import urllib.request +from collections.abc import Callable, Iterable, Iterator, Mapping + +from .manifests import Model + +__all__ = ["ALREADY_STARTED", "START_PATH", "Router", "StartError", "credentials", "open_router", "redact"] + +START_PATH = "/ousast/v1/start" +ALREADY_STARTED = "already started on this boot" # the runner's 409 when its command is running +NAMESPACE = "ate-system" +SERVICE = "svc/atenet-router" +TARGET_HEADER = "ate-target-actor" +RETRY_STATUS = frozenset({502, 503, 504}) +_FORWARDING = re.compile(r"Forwarding from 127\.0\.0\.1:(\d+)") + + +class StartError(RuntimeError): + """The start request was refused, or no actor answered it in time; the message carries the response text.""" + + +def redact(text: str, secrets: Iterable[str]) -> str: + """``text`` with every non-empty secret value replaced by ``[redacted]``.""" + for secret in secrets: + if secret: + text = text.replace(secret, "[redacted]") + return text + + +def credentials(model: Model | None, environ: Mapping[str, str] | None = None) -> dict[str, str]: + """The provider credentials of a task bound to ``model``: ``{secretKey.key: value}`` from the environment. + + A task without a Model, or a Model without ``secretKey``, gets none. A named variable that is unset or empty + raises :class:`StartError` naming the variable (never a value). + """ + if model is None or model.secret_key is None: + return {} + env = os.environ if environ is None else environ + name = model.secret_key.key + value = env.get(name, "") + if not value: + where = f"secretKey {model.secret_key.name}; it is not set in this environment or .env" + raise StartError(f"Model/{model.metadata.name} needs {name} ({where})") + return {name: value} + + +class Router: + """Posts start requests to one task actor at a time through the router at ``url``.""" + + def __init__(self, url: str) -> None: + self.url = url.rstrip("/") + + def starter(self, atespace: str, task: str, run: str, credentials: Mapping[str, str]) -> Callable[[], str | None]: + """:meth:`start_task` bound to one task, returning the refusal text instead of raising (None once accepted). + + The reconciler calls it again after a re-resume, since a resumed actor may have rebooted and wait for a + start; on those later calls a 409 saying the command is already started on this boot means the process + tree survived and the command is still running, which is not a failure. + """ + calls = 0 + + def start() -> str | None: + nonlocal calls + calls += 1 + try: + self.start_task(atespace, task, run, credentials) + except StartError as exc: + return None if calls > 1 and ALREADY_STARTED in str(exc) else str(exc) + return None + + return start + + def start_task(self, atespace: str, task: str, run: str, credentials: Mapping[str, str]) -> None: + """Start ``/``'s command once; :class:`StartError` on 409, another 4xx, or the timeout.""" + body = json.dumps({"run": run, "task": task, "credentials": dict(credentials)}).encode("utf-8") + headers = {"Content-Type": "application/json", TARGET_HEADER: f"{atespace}/{task}"} + limit = float(os.environ.get("OUSAST_START_TIMEOUT") or 300) + deadline, backoff, last = time.monotonic() + limit, 0.5, "no attempt made" + secrets = list(credentials.values()) + while True: + request = urllib.request.Request(self.url + START_PATH, data=body, method="POST", headers=headers) + try: + with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the router's local URL + if 200 <= int(response.status) < 300: + return + last = f"HTTP {response.status}" + except urllib.error.HTTPError as exc: + text = redact(exc.read().decode("utf-8", errors="replace").strip()[:400], secrets) + if exc.code not in RETRY_STATUS: + raise StartError(f"start of {atespace}/{task} refused (HTTP {exc.code}): {text}") from None + last = f"HTTP {exc.code}: {text}" + except (urllib.error.URLError, http.client.HTTPException, OSError) as exc: + last = redact(f"{type(exc).__name__}: {exc}", secrets) + if time.monotonic() + backoff > deadline: + raise StartError(f"start of {atespace}/{task} not accepted within {limit:g}s (OUSAST_START_TIMEOUT): {last}") + time.sleep(backoff) + backoff = min(backoff * 2, 10.0) + + +def _drain(stream: Iterable[str], lines: queue.Queue[str | None]) -> None: + for line in stream: # kubectl logs a line per proxied connection; an unread pipe would block it + lines.put(line) + lines.put(None) + + +@contextlib.contextmanager +def open_router(context: str | None = None, *, kubectl: str = "kubectl", ready_timeout: float = 30.0) -> Iterator[Router]: + """A :class:`Router` for the duration of the block. + + ``OUSAST_ROUTER_URL`` set: that URL, nothing started. Otherwise ``kubectl port-forward svc/atenet-router :80`` + in ``ate-system`` of ``context`` (default ``kind-$KIND_CLUSTER_NAME``, ``kind-ousast``), the local port read + from kubectl's ``Forwarding from 127.0.0.1:NNNN`` line; on exit that child process is terminated. + """ + override = os.environ.get("OUSAST_ROUTER_URL") + if override: + yield Router(override) + return + context = context or "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast") + argv = [kubectl, "--context", context, "-n", NAMESPACE, "port-forward", SERVICE, ":80"] + proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + try: + assert proc.stdout is not None + lines: queue.Queue[str | None] = queue.Queue() + threading.Thread(target=_drain, args=(proc.stdout, lines), name="router-port-forward", daemon=True).start() + deadline, seen, port = time.monotonic() + ready_timeout, list[str](), None + while port is None: + try: + line = lines.get(timeout=max(deadline - time.monotonic(), 0.01)) + except queue.Empty: + raise StartError(f"kubectl port-forward printed no port within {ready_timeout:g}s: {''.join(seen)[-400:]}") from None + if line is None: + raise StartError(f"kubectl port-forward exited ({proc.wait()}): {''.join(seen)[-400:]}") + seen.append(line) + match = _FORWARDING.search(line) + port = int(match.group(1)) if match else None + yield Router(f"http://127.0.0.1:{port}") + finally: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() diff --git a/src/openultrasast/plane/runner.py b/src/openultrasast/plane/runner.py index f81eb8a..0ac6145 100644 --- a/src/openultrasast/plane/runner.py +++ b/src/openultrasast/plane/runner.py @@ -7,6 +7,16 @@ metadata paths on port 80, ``/readyz`` answering 503 until every workspace is ma Workspaces are prepared on the first boot only: ``/workspace`` survives suspend and resume, the process tree does not (ax docs/runner.md). +The command never starts by itself (Requirements 4.5, 4.6). Agent Substrate boots every new ActorTemplate once as a +"golden" actor to snapshot it, with this image and the same ``AX_TASK_YAML``/``AX_WORKSPACES_YAML``/env, and inside +the sandbox nothing tells that boot from the real one. So after the workspaces are ready the runner waits for +``POST /ousast/v1/start`` with ``{"run", "task", "credentials"}``, which the reconciler sends through Substrate's +router to the task's actor only (``plane/router.py``). ``run`` and ``task`` must match ``OUSAST_RUN`` and the Task's +name (409 otherwise); one request per boot is accepted (202), later ones are 409, and a boot whose completion marker +records a delivered run refuses every request (409 with the reason). Before the workspaces are ready the answer is +503, which the reconciler retries. The credentials live in this process's memory and go into the command's +environment only; they are never logged or written, and their values are redacted from the command's echoed stderr. + The command runs as a child process, ``python -m openultrasast.plane.tasks. ``, in its own process group, with the first workspace as its working directory and ``AX_METADATA_URL`` pointing at this server. ax has no artifact channel and never reports that a command finished, so after the child exits the @@ -25,6 +35,8 @@ Environment the runner reads, all optional except the two ax variables: AX_RUNNER_HTTP "0" disables the health server (unit tests only) AX_RUNNER_PORT port of the health server, default 80; "0" picks a free port AX_RUNNER_EXIT_AFTER_COMMAND "1" exits after delivery with 0 done, 2 failed, 3 unfinished (unit tests only) + AX_RUNNER_AUTOSTART "1" runs the command without waiting for a start request (unit tests only; + required with AX_RUNNER_HTTP=0, where no request could arrive) AX_RUNNER_TASK_PACKAGE package the command's module is looked up in (unit tests only) OUSAST_STATE_DIR completion markers, default ``/workspace/.ousast-state`` (the durable volume) OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10 @@ -64,17 +76,22 @@ from typing import Any import yaml from openultrasast.plane.manifests import GitSource, ManifestError, Task, Workspace, parse_manifest +from openultrasast.plane.router import ALREADY_STARTED +from openultrasast.plane.router import redact as _redact __all__ = [ "DEFAULT_PORT", "DELIVERY_ATTEMPTS", "EXIT_BY_STATUS", + "START_PATH", "STATE_DIR", "TASK_PACKAGE", "Child", "DeliveryError", "HealthServer", "RunnerError", + "Start", + "StartGate", "build_tar", "deliver", "clone_destination", @@ -173,9 +190,105 @@ def load_workspaces(text: str) -> dict[str, Workspace]: # --- health server ----------------------------------------------------------------------------------------------- +START_PATH = "/ousast/v1/start" +_CREDENTIAL_NAME = re.compile(r"[A-Z][A-Z0-9_]*") +_RESERVED_PREFIXES = ("AX_", "OUSAST_") # the runner's own contract; a credential must not redirect it + + +@dataclass(frozen=True) +class Start: + """An accepted start request: the credentials go to the command's environment and nowhere else.""" + + credentials: Mapping[str, str] + + def __repr__(self) -> str: # never print a value by accident + return f"Start(credentials={sorted(self.credentials)})" + + +class StartGate: + """The one start request per boot (Requirements 4.5, 4.6). + + Before :meth:`arm` (workspaces not ready) a request is answered 503, which the reconciler retries; afterwards + one request whose ``run`` and ``task`` match the Task is accepted (202), every other one is 409, as is any + request once :meth:`refuse` recorded why this boot runs nothing (a delivered run). The request body is never + logged, and no answer echoes it. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._accepted = threading.Event() + self._expected: tuple[str, str] | None = None + self._refusal: str | None = None + self._start: Start | None = None + + def arm(self, run: str, task: str) -> None: + with self._lock: + self._expected = (run, task) + + def refuse(self, reason: str) -> None: + with self._lock: + self._refusal = reason + + def autostart(self) -> Start: + """The test escape ``AX_RUNNER_AUTOSTART=1``: start without a request; a later request is refused.""" + with self._lock: + self._start = Start({}) + self._accepted.set() + return self._start + + def offer(self, body: bytes) -> tuple[int, str]: + with self._lock: + if self._refusal is not None: + return 409, self._refusal + if self._accepted.is_set(): + return 409, f"this task was {ALREADY_STARTED}" + if self._expected is None: + return 503, "workspaces not materialised" + try: + request = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + return 400, "the start body is not JSON" + if not isinstance(request, dict): + return 400, "the start body must be a JSON object" + creds = request.get("credentials") or {} + if not isinstance(creds, dict) or not all( + isinstance(k, str) and _CREDENTIAL_NAME.fullmatch(k) and not k.startswith(_RESERVED_PREFIXES) and isinstance(v, str) + for k, v in creds.items() + ): + return 400, "credentials must map ^[A-Z][A-Z0-9_]*$ names (not AX_/OUSAST_) to strings" + run, task = self._expected + if request.get("run") != run or request.get("task") != task: + return 409, f"this actor runs task {task!r} of run {run!r}" + self._start = Start(dict(creds)) + self._accepted.set() + return 202, "started" + + def wait(self, stop: threading.Event) -> Start | None: + """The accepted start, or None once ``stop`` is set first.""" + while not self._accepted.wait(0.1): + if stop.is_set(): + return None + return self._start + + class _HealthHandler(BaseHTTPRequestHandler): server: HealthServer + def do_POST(self) -> None: # noqa: N802 - http.server's naming + if self.path.split("?", 1)[0] != START_PATH: + self._answer(404, "not found") + return + try: + length = int(self.headers.get("Content-Length") or 0) + except ValueError: + length = -1 + if not 0 <= length <= 1 << 20: + self._answer(400, "bad Content-Length") + return + status, text = self.server.gate.offer(self.rfile.read(length)) + log.info("start request: %d %s", status, text) + self._answer(status, text) + def do_GET(self) -> None: # noqa: N802 - http.server's naming path = self.path.split("?", 1)[0] # ax's controller probes /readyz?check=workspace if path == "/healthz": @@ -202,14 +315,15 @@ class _HealthHandler(BaseHTTPRequestHandler): class HealthServer(ThreadingHTTPServer): """``/healthz`` always 200; ``/readyz`` 503 until ``ready`` is set; the Task and Workspaces YAML under - ``/metadata/v1alpha1/ax/``. Serves in a daemon thread.""" + ``/metadata/v1alpha1/ax/``; ``POST /ousast/v1/start`` goes to ``gate``. Serves in a daemon thread.""" daemon_threads = True - def __init__(self, port: int, ready: threading.Event, metadata: Mapping[str, str] | None = None) -> None: + def __init__(self, port: int, ready: threading.Event, metadata: Mapping[str, str] | None = None, gate: StartGate | None = None) -> None: super().__init__(("0.0.0.0", port), _HealthHandler) self.ready = ready self.metadata = dict(metadata or {}) + self.gate = gate or StartGate() self._thread = threading.Thread(target=self.serve_forever, name="ax-health", daemon=True) @property @@ -225,9 +339,11 @@ class HealthServer(ThreadingHTTPServer): self._thread.join(timeout=5) -def start_health_server(port: int, ready: threading.Event, metadata: Mapping[str, str] | None = None) -> HealthServer: - """Serve ``/healthz``, ``/readyz`` and ax's metadata paths (``metadata`` keyed ``task``/``workspaces``).""" - server = HealthServer(port, ready, metadata) +def start_health_server( + port: int, ready: threading.Event, metadata: Mapping[str, str] | None = None, gate: StartGate | None = None +) -> HealthServer: + """Serve ``/healthz``, ``/readyz``, ax's metadata paths (``metadata`` keyed ``task``/``workspaces``) and the start.""" + server = HealthServer(port, ready, metadata, gate) server.start() return server @@ -426,7 +542,8 @@ def task_module(command: Sequence[str], package: str = TASK_PACKAGE) -> str: class Child: """``spec.command`` as a child in its own process group; its stderr is echoed and the tail kept.""" - def __init__(self, argv: Sequence[str], cwd: str | None, env: Mapping[str, str]) -> None: + def __init__(self, argv: Sequence[str], cwd: str | None, env: Mapping[str, str], secrets: Sequence[str] = ()) -> None: + self.secrets = [s for s in secrets if s] self.proc = subprocess.Popen(list(argv), cwd=cwd, env=dict(env), stderr=subprocess.PIPE, start_new_session=True) self.tail: deque[str] = deque(maxlen=STDERR_TAIL_LINES) self._reader = threading.Thread(target=self._echo, name="task-stderr", daemon=True) @@ -435,7 +552,7 @@ class Child: def _echo(self) -> None: assert self.proc.stderr is not None for raw in self.proc.stderr: - line = raw.decode("utf-8", errors="replace") + line = _redact(raw.decode("utf-8", errors="replace"), self.secrets) sys.stderr.write(line) sys.stderr.flush() self.tail.append(line.rstrip("\n")) @@ -484,15 +601,18 @@ def summary_after(code: int, output_dir: Path, name: str, stderr_tail: Sequence[ def run_command( - task: Task, output_dir: Path, env: Mapping[str, str], stop: threading.Event, grace: float + task: Task, output_dir: Path, env: Mapping[str, str], stop: threading.Event, grace: float, credentials: Mapping[str, str] | None = None ) -> tuple[int | None, dict[str, Any]]: """Run ``python -m . `` in the first workspace; (exit code, summary). - (None, {}) when ``stop`` interrupted it: the command did not finish and a resume runs it again. + ``credentials`` (from the start request) are added to the child's environment only, and their values are + redacted from its echoed stderr and from the stderr tail a crash summary keeps. (None, {}) when ``stop`` + interrupted it: the command did not finish and a resume runs it again. """ + credentials = dict(credentials or {}) module = task_module(task.command, env.get("AX_RUNNER_TASK_PACKAGE") or TASK_PACKAGE) cwd = task.workspaces[0].path if task.workspaces else None - child = Child([sys.executable, "-m", module, *task.command[1:]], cwd, env) + child = Child([sys.executable, "-m", module, *task.command[1:]], cwd, {**env, **credentials}, list(credentials.values())) code = child.wait(stop, grace) if code is None: return None, {} @@ -611,8 +731,9 @@ def _finish(session: _Session, env: Mapping[str, str], exit_code: int | None, su return marker -def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threading.Event) -> int: - """Materialise, run the command once, deliver; on a resumed volume, only what the marker says is left.""" +def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threading.Event, gate: StartGate) -> int: + """Materialise, wait for the start request, run the command once, deliver; on a resumed volume, only what the + marker says is left. A boot whose marker records a delivered run refuses every start request.""" session: _Session | None = None try: session = _prepare(env) @@ -622,11 +743,21 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa cache_dir = env.get("OUSAST_CASE_CACHE") materialise(session.task, session.workspaces, Path(cache_dir) if cache_dir else None, load_pins(env.get("OUSAST_GIT_PINS", ""))) session.prepared.touch() - ready.set() done = _read_marker(session.marker) if done is not None and done.get("delivered"): + gate.refuse(f"task {session.task.metadata.name} already ran and was delivered ({done.get('status')})") + ready.set() log.info("task %s already ran and was delivered (%s); not running it again", session.task.metadata.name, done.get("status")) return _exit_of(done) + gate.arm(env.get("OUSAST_RUN", ""), session.task.metadata.name) + ready.set() + if env.get("AX_RUNNER_AUTOSTART") == "1": + start: Start | None = gate.autostart() + else: + log.info("workspaces ready; waiting for the start request on %s", START_PATH) + start = gate.wait(stop) + if start is None: + return 0 if done is not None: log.info("task %s already ran; retrying the delivery only", session.task.metadata.name) stored = done.get("summary") @@ -636,17 +767,26 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa if stop.is_set(): return 0 log.info("workspaces ready; running task %s: %s", session.task.metadata.name, list(session.task.command)) - code, summary = run_command(session.task, session.output_dir, env, stop, float(env.get("OUSAST_TERM_GRACE") or TERM_GRACE)) + grace = float(env.get("OUSAST_TERM_GRACE") or TERM_GRACE) + code, summary = run_command(session.task, session.output_dir, env, stop, grace, start.credentials) if code is None: log.info("task %s was stopped before its command finished; a resume runs it again", session.task.metadata.name) return 0 return _exit_of(_finish(session, env, code, summary)) except RunnerError as exc: + # The failure is reported, never acted on unasked: a start request is answered with it (409 without a + # session, else 202 and the failed summary is delivered), so the golden boot delivers nothing either. log.error("runner failed: %s", exc) if session is None: + gate.refuse(f"runner failed: {exc}") + ready.set() return EXIT_BY_STATUS["failed"] summary = _failed(f"runner failed: {exc}", read_summary(session.output_dir)) write_summary(session.output_dir, summary) + gate.arm(env.get("OUSAST_RUN", ""), session.task.metadata.name) + ready.set() + if env.get("AX_RUNNER_AUTOSTART") != "1" and gate.wait(stop) is None: + return 0 return _exit_of(_finish(session, env, None, summary)) @@ -667,17 +807,20 @@ def main(argv: Sequence[str] | None = None, environ: MutableMapping[str, str] | env = os.environ if environ is None else environ if argv: log.warning("ax-task-runner ignores command-line arguments %r; the Task manifest is the only input", list(argv)) - ready, stop = threading.Event(), threading.Event() + ready, stop, gate = threading.Event(), threading.Event(), StartGate() previous = _on_sigterm(stop) server: HealthServer | None = None if env.get("AX_RUNNER_HTTP", "1") != "0": metadata = {"task": env.get("AX_TASK_YAML", ""), "workspaces": env.get("AX_WORKSPACES_YAML", "")} - server = start_health_server(int(env.get("AX_RUNNER_PORT", str(DEFAULT_PORT))), ready, metadata) + server = start_health_server(int(env.get("AX_RUNNER_PORT", str(DEFAULT_PORT))), ready, metadata, gate) env["AX_RUNNER_BOUND_PORT"] = str(server.port) env["AX_METADATA_URL"] = f"http://127.0.0.1:{server.port}" log.info("health and metadata server on port %d", server.port) try: - code = _run_once(env, ready, stop) + if server is None and env.get("AX_RUNNER_AUTOSTART") != "1": + log.error("AX_RUNNER_HTTP=0 without AX_RUNNER_AUTOSTART=1: no start request could ever arrive") + return EXIT_BY_STATUS["failed"] + code = _run_once(env, ready, stop, gate) if stop.is_set(): return 0 if env.get("AX_RUNNER_EXIT_AFTER_COMMAND") == "1": diff --git a/tests/plane_stub_task.py b/tests/plane_stub_task.py index ee26d09..1d2b967 100644 --- a/tests/plane_stub_task.py +++ b/tests/plane_stub_task.py @@ -3,11 +3,13 @@ The tests copy it into a throwaway package (``AX_RUNNER_TASK_PACKAGE``) so the runner starts it the way it starts a real task, ``python -m .stub_task``. ``STUB_MODE`` picks the outcome (``done``, ``failed``, ``unfinished``, ``crash``, ``nosummary``, ``exit``, ``sleep``); ``STUB_COUNTER`` names a file that gets one line -per run. +per run. ``STUB_SECRET_VAR`` names a credential variable: the stub records the SHA-256 of its value (or None), +never the value. """ from __future__ import annotations +import hashlib import json import os import signal @@ -23,6 +25,11 @@ def _get(url: str) -> tuple[int, str]: return int(response.status), response.read().decode("utf-8") +def _secret_hash(name: str | None) -> str | None: + value = os.environ.get(name) if name else None + return hashlib.sha256(value.encode("utf-8")).hexdigest() if value else None + + def main(argv: list[str]) -> None: out = Path(os.environ["OUSAST_OUTPUT_DIR"]) out.mkdir(parents=True, exist_ok=True) @@ -45,6 +52,7 @@ def main(argv: list[str]) -> None: "pgid": os.getpgid(0), "metadata_url": metadata, "metadata_task": _get(f"{metadata}/metadata/v1alpha1/ax/task")[1] if metadata else None, + "secret_sha256": _secret_hash(os.environ.get("STUB_SECRET_VAR")), } (out / "facts.json").write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8") if mode == "sleep": # ignore SIGTERM so only the runner's SIGKILL ends the group; the grandchild inherits it @@ -52,6 +60,9 @@ def main(argv: list[str]) -> None: grandchild = subprocess.Popen(["sleep", "60"]) (out / "pids.json").write_text(json.dumps([os.getpid(), grandchild.pid]), encoding="utf-8") time.sleep(60) + if mode == "leak": # a task that prints its credential and crashes: the runner must redact it + sys.stderr.write(f"auth failed for key {os.environ.get(os.environ.get('STUB_SECRET_VAR', ''), '')}\n") + raise SystemExit(5) if mode == "crash": raise RuntimeError("stub exploded") if mode == "exit": diff --git a/tests/test_plane_reconciler.py b/tests/test_plane_reconciler.py index 7229bdc..baa24c6 100644 --- a/tests/test_plane_reconciler.py +++ b/tests/test_plane_reconciler.py @@ -5,7 +5,8 @@ internal/controller/reconciler.go, internal/server/server.go): ``apply`` creates fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase -- and on the scripted poll it POSTs a scripted tar to the ``OUSAST_ARTIFACT_URL`` in the applied Task's env, or -turns ``Failed`` with a condition message, or vanishes. Every outcome comes from a scripted delivery. +turns ``Failed`` with a condition message, or vanishes. Every outcome comes from a scripted delivery, and a +delivery happens only after the reconciler's start request reached the fake router (Req 4.5). """ from __future__ import annotations @@ -16,7 +17,10 @@ import stat import subprocess import sys import tarfile +import threading import time +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import pytest @@ -27,6 +31,7 @@ from openultrasast.plane.workspaces import workspaces RECONCILER = Path("src/openultrasast/plane/reconciler.py") PIN = "0123456789abcdef0123456789abcdef01234567" +SECRET = "sk-reconciler-test-7a2b-only-in-the-start-request" # The fields ax's API server decodes (pkg/apis/v1alpha1/ax.proto, protojson: an unknown field is an error at # ``ax apply``). ``None`` marks a value ax types as free-form (a Struct) or that this test does not descend into. @@ -114,6 +119,9 @@ elif argv[:2] == ["get", "task"]: rec.update(phase=script["phase"], message=script.get("message", "")) if script.get("vanish"): rec["gone"] = True + started = (HERE / "started" / argv[2]).exists() # the runner runs its command only after the start request + if rec["phase"] == "Running" and rec["polls"] >= script.get("polls", 1) and started and not rec.get("sent"): + rec["sent"] = True if script.get("deliver", True) and "phase" not in script and not script.get("vanish"): buf = io.BytesIO() with tarfile.open(fileobj=buf, mode="w") as tar: @@ -152,6 +160,7 @@ metadata: spec: provider: deepseek model: deepseek-chat + secretKey: {name: deepseek, key: DEEPSEEK_API_KEY} parameters: {cache_hit_per_m: 0.07, input_per_m: 0.27, output_per_m: 1.1} --- apiVersion: ax.io/v1alpha1 @@ -234,6 +243,8 @@ SCRIPT = { class Fake: + router: FakeRouter + def __init__(self, root: Path) -> None: self.root = root self.ax = root / "ax" @@ -263,15 +274,61 @@ class Fake: return out +class FakeRouter(ThreadingHTTPServer): + """Agent Substrate's router as the reconciler sees it: records each start request and answers from ``codes`` + (by ax task name, default 202); an accepted start marks the task started for the fake ax, whose runner only + then runs its command and delivers.""" + + daemon_threads = True + + def __init__(self, root: Path) -> None: + super().__init__(("127.0.0.1", 0), _RouterHandler) + self.root = root + self.codes: dict[str, tuple[int, str]] = {} + self.starts: list[dict] = [] + threading.Thread(target=self.serve_forever, daemon=True).start() + + +class _RouterHandler(BaseHTTPRequestHandler): + server: FakeRouter + + def do_POST(self) -> None: # noqa: N802 + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0))) + target = self.headers.get("ate-target-actor", "") + name = target.partition("/")[2] + code, text = self.server.codes.get(name, (202, "started")) + self.server.starts.append({"at": time.monotonic(), "path": self.path, "target": target, "body": body, "code": code}) + if code == 202: + (self.server.root / "started").mkdir(exist_ok=True) + (self.server.root / "started" / name).touch() + with (self.server.root / "ax.log").open("a") as log: + log.write(f"{time.monotonic():.4f} started {name} {code}\n") + self.send_response(code) + self.send_header("Content-Length", str(len(text.encode()))) + self.end_headers() + self.wfile.write(text.encode()) + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + pass + + @pytest.fixture -def fake(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Fake: +def fake(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Fake]: monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path / "results")) monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1") monkeypatch.setenv("OUSAST_POLL_SECONDS", "0.02") monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "20") # a test that never delivers fails in seconds, not in 2 hours + monkeypatch.setenv("OUSAST_START_TIMEOUT", "2") + monkeypatch.setenv("DEEPSEEK_API_KEY", SECRET) # the Model's secretKey variable, as ``.env`` would set it fake = Fake(tmp_path) fake.script(SCRIPT) - return fake + fake.router = FakeRouter(tmp_path) + monkeypatch.setenv("OUSAST_ROUTER_URL", f"http://127.0.0.1:{fake.router.server_address[1]}") + try: + yield fake + finally: + fake.router.shutdown() + fake.router.server_close() def write_run(tmp_path: Path, name: str, tasks: str = CHAIN) -> Path: @@ -426,6 +483,54 @@ def test_no_delivery_within_the_task_timeout_is_failed(fake: Fake, tmp_path: Pat assert [n for _, n in fake.events("delete")] == ["silent-repo-facts"], "ax delete follows every task" +def files_containing(root: Path, needle: str) -> list[str]: + return sorted(str(p) for p in root.rglob("*") if p.is_file() and needle.encode() in p.read_bytes()) + + +def test_start_is_sent_once_after_resume_with_the_models_credential(fake: Fake, tmp_path: Path) -> None: + """Req 4.5/4.6: one start per task through the router, after ax accepted a resume; the bound Model's credential + travels only in that request, never in the rendered Task, the run dir or any file the run leaves behind.""" + assert reconciler.run(write_run(tmp_path, "sig"), ax=str(fake.ax)) == "done" + starts = fake.router.starts + assert [s["target"] for s in starts] == ["default/sig-repo-facts", "default/sig-verify", "default/sig-agree"], "once each" + assert all(s["path"] == "/ousast/v1/start" and s["code"] == 202 for s in starts) + by_task = {s["body"]["task"]: s for s in starts} + assert by_task["sig-verify"]["body"] == {"run": "sig", "task": "sig-verify", "credentials": {"DEEPSEEK_API_KEY": SECRET}} + assert by_task["sig-repo-facts"]["body"]["credentials"] == {} and by_task["sig-agree"]["body"]["credentials"] == {}, "no Model" + accepted = {} + for stamp, words in fake.log(): # the last resume of each task is the one ax accepted + if words[:2] == ["resume", "task"]: + accepted[words[2]] = stamp + delivered = {name: stamp for stamp, name in fake.events("delivered")} + for s in starts: + name = s["body"]["task"] + assert accepted[name] < s["at"] < delivered[name.removeprefix("sig-")], f"{name}: resume, then start, then delivery" + verify_model = fake.applied("sig", "verify")["docs"][1] + assert verify_model["spec"]["secretKey"] == {"name": "deepseek", "key": "DEEPSEEK_API_KEY"}, "the manifest names the key" + assert "DEEPSEEK_API_KEY" not in fake.applied("sig", "verify")["env"], "the rendered Task carries no credential" + rendered = str(reconciler.run_dir("sig") / "verify" / "task.yaml") + assert rendered in files_containing(tmp_path, "OUSAST_ARTIFACT_URL"), "the scan reads the rendered files it vouches for" + assert files_containing(tmp_path, SECRET) == [], "no rendered task.yaml, state, log or applied record holds the value" + + +def test_a_model_bound_task_without_its_secret_fails_before_ax_sees_it(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DEEPSEEK_API_KEY") + assert reconciler.run(write_run(tmp_path, "nokey"), ax=str(fake.ax)) == "failed" + assert statuses("nokey") == {"repo-facts": "done", "verify": "failed", "agree": "pending"} + reason = state_of("nokey")["tasks"]["verify"]["reason"] + assert "DEEPSEEK_API_KEY" in reason and "deepseek-flash" in reason + assert [name for _, name in fake.events("apply")] == ["repo-facts"], "verify is never applied without its credential" + + +def test_a_refused_start_fails_the_task_with_the_runners_answer(fake: Fake, tmp_path: Path) -> None: + fake.router.codes["ref-repo-facts"] = (409, "task ref-repo-facts already ran and was delivered (done)") + assert reconciler.run(write_run(tmp_path, "ref"), ax=str(fake.ax)) == "failed" + reason = state_of("ref")["tasks"]["repo-facts"]["reason"] + assert "HTTP 409" in reason and "already ran and was delivered" in reason + assert len(fake.router.starts) == 1, "a refusal is final, not retried" + assert [n for _, n in fake.events("delete")] == ["ref-repo-facts"], "ax delete follows a refused start" + + def test_outcome_of_needs_a_delivered_summary() -> None: assert reconciler.outcome_of(None) == ("failed", "no summary.json delivered") assert reconciler.outcome_of({"status": "done", "units_done": 1, "units_total": 1}) == ("done", "") diff --git a/tests/test_plane_router.py b/tests/test_plane_router.py new file mode 100644 index 0000000..1ff0453 --- /dev/null +++ b/tests/test_plane_router.py @@ -0,0 +1,189 @@ +"""The start signal through Agent Substrate's router (ai-service-plane Requirements 4.5, 4.6): port-forward +lifecycle with a fake kubectl, retry and refusal against a fake router, credentials from the environment.""" + +from __future__ import annotations + +import json +import stat +import sys +import threading +import time +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +from openultrasast.plane.manifests import Metadata, Model, SecretKey +from openultrasast.plane.router import START_PATH, StartError, credentials, open_router + +SECRET = "sk-router-test-5d1e-never-logged" + +FAKE_KUBECTL = """#!{python} +import json, os, sys, time +from pathlib import Path +here = Path(__file__).resolve().parent +(here / "kubectl.json").write_text(json.dumps({{"argv": sys.argv[1:], "pid": os.getpid()}})) +mode = os.environ.get("FAKE_KUBECTL_MODE", "ok") +if mode == "fail": + print('error: services "atenet-router" not found', flush=True) + sys.exit(1) +print("Forwarding from 127.0.0.1:{port} -> 80", flush=True) +print("Forwarding from [::1]:{port} -> 80", flush=True) +time.sleep(600) +""" + + +class FakeRouter(ThreadingHTTPServer): + """Answers the start path from ``codes`` in turn (the last one repeats) and records every request.""" + + daemon_threads = True + + def __init__(self, codes: list[tuple[int, str]]) -> None: + super().__init__(("127.0.0.1", 0), _RouterHandler) + self.codes = codes + self.requests: list[tuple[str, dict[str, str], dict]] = [] + threading.Thread(target=self.serve_forever, daemon=True).start() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + +class _RouterHandler(BaseHTTPRequestHandler): + server: FakeRouter + + def do_POST(self) -> None: # noqa: N802 + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0))) + self.server.requests.append((self.path, {k.lower(): v for k, v in self.headers.items()}, body)) + code, text = self.server.codes[min(len(self.server.requests), len(self.server.codes)) - 1] + payload = text.encode() + self.send_response(code) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + pass + + +@pytest.fixture +def fake_router(request: pytest.FixtureRequest) -> Iterator[FakeRouter]: + server = FakeRouter(getattr(request, "param", [(202, "started")])) + try: + yield server + finally: + server.shutdown() + server.server_close() + + +def write_kubectl(tmp_path: Path, port: int) -> Path: + path = tmp_path / "kubectl" + path.write_text(FAKE_KUBECTL.format(python=sys.executable, port=port), encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + return path + + +def alive(pid: int) -> bool: + try: + return Path(f"/proc/{pid}/stat").read_text().split(")")[-1].split()[0] != "Z" + except (FileNotFoundError, IndexError): + return False + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key in ("OUSAST_ROUTER_URL", "OUSAST_START_TIMEOUT", "KIND_CLUSTER_NAME", "FAKE_KUBECTL_MODE"): + monkeypatch.delenv(key, raising=False) + + +def test_port_forward_port_is_parsed_and_the_child_is_terminated_on_exit(tmp_path: Path, fake_router: FakeRouter) -> None: + port = int(fake_router.server_address[1]) + kubectl = write_kubectl(tmp_path, port) + with open_router(kubectl=str(kubectl)) as router: + assert router.url == f"http://127.0.0.1:{port}" + seen = json.loads((tmp_path / "kubectl.json").read_text()) + assert seen["argv"] == ["--context", "kind-ousast", "-n", "ate-system", "port-forward", "svc/atenet-router", ":80"] + assert alive(seen["pid"]) + router.start_task("default", "run-t", "run", {}) + deadline = time.monotonic() + 5 + while alive(seen["pid"]) and time.monotonic() < deadline: + time.sleep(0.05) + assert not alive(seen["pid"]), "the port-forward child is terminated when the block ends" + assert fake_router.requests[0][1]["ate-target-actor"] == "default/run-t" + + +def test_a_failing_port_forward_raises_with_kubectls_output(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FAKE_KUBECTL_MODE", "fail") + kubectl = write_kubectl(tmp_path, 1) + with pytest.raises(StartError, match="atenet-router.*not found"), open_router("kind-other", kubectl=str(kubectl)): + pass + assert json.loads((tmp_path / "kubectl.json").read_text())["argv"][1] == "kind-other" + + +def test_router_url_override_starts_no_port_forward(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None: + monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url) + with open_router(kubectl=str(tmp_path / "no-such-kubectl")) as router: + assert router.url == fake_router.url + assert not (tmp_path / "kubectl.json").exists() + + +@pytest.mark.parametrize("fake_router", [[(503, "no actor yet"), (502, "bad gateway"), (202, "started")]], indirect=True) +def test_start_retries_503_and_502_then_succeeds(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None: + monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url) + with open_router() as router: + router.start_task("default", "chain-verify", "chain", {"DEEPSEEK_API_KEY": SECRET}) + assert len(fake_router.requests) == 3 + path, headers, body = fake_router.requests[-1] + assert path == START_PATH and headers["ate-target-actor"] == "default/chain-verify" + assert body == {"run": "chain", "task": "chain-verify", "credentials": {"DEEPSEEK_API_KEY": SECRET}} + + +@pytest.mark.parametrize("fake_router", [[(409, f"this actor runs task 'x' (echo {SECRET})")]], indirect=True) +def test_a_409_is_raised_at_once_with_the_response_text_redacted(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None: + monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url) + with open_router() as router, pytest.raises(StartError, match="HTTP 409.*runs task 'x'") as caught: + router.start_task("default", "t", "r", {"DEEPSEEK_API_KEY": SECRET}) + assert len(fake_router.requests) == 1 and SECRET not in str(caught.value) + assert router.starter("default", "t", "r", {})() is not None, "the starter returns the refusal instead of raising" + + +@pytest.mark.parametrize("fake_router", [[(409, "this task was already started on this boot")]], indirect=True) +def test_a_restart_after_re_resume_accepts_a_running_command(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None: + """First start: 'already started' is a refusal. After a re-resume the actor may not have rebooted; then the + runner's 'already started on this boot' means the command still runs, not a failure.""" + monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url) + with open_router() as router: + start = router.starter("default", "t", "r", {}) + assert start() is not None, "a first start that finds the command running is a refusal" + assert start() is None, "a re-sent start after a re-resume is not" + fake_router.codes = [(409, "task t already ran and was delivered (done)")] + assert start() is not None, "any other refusal stays one" + + +@pytest.mark.parametrize("fake_router", [[(503, "workspaces not materialised")]], indirect=True) +def test_start_gives_up_after_the_start_timeout(monkeypatch: pytest.MonkeyPatch, fake_router: FakeRouter) -> None: + monkeypatch.setenv("OUSAST_ROUTER_URL", fake_router.url) + monkeypatch.setenv("OUSAST_START_TIMEOUT", "1") + with open_router() as router, pytest.raises(StartError, match="OUSAST_START_TIMEOUT.*workspaces not materialised"): + router.start_task("default", "t", "r", {}) + assert len(fake_router.requests) >= 2 + + +def test_connection_errors_are_retried_until_the_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OUSAST_ROUTER_URL", "http://127.0.0.1:9") + monkeypatch.setenv("OUSAST_START_TIMEOUT", "1") + with open_router() as router, pytest.raises(StartError, match="within 1s"): + router.start_task("default", "t", "r", {}) + + +def model(secret: SecretKey | None) -> Model: + return Model(metadata=Metadata(name="deepseek-flash"), provider="deepseek", model="deepseek-flash", secret_key=secret) + + +def test_credentials_come_from_the_models_secret_key_variable() -> None: + key = SecretKey(name="deepseek", key="DEEPSEEK_API_KEY") + assert credentials(None, {}) == {} and credentials(model(None), {"DEEPSEEK_API_KEY": SECRET}) == {} + assert credentials(model(key), {"DEEPSEEK_API_KEY": SECRET}) == {"DEEPSEEK_API_KEY": SECRET} + with pytest.raises(StartError, match="needs DEEPSEEK_API_KEY"): + credentials(model(key), {"DEEPSEEK_API_KEY": ""}) diff --git a/tests/test_plane_runner.py b/tests/test_plane_runner.py index b4ca7cf..d9e7df4 100644 --- a/tests/test_plane_runner.py +++ b/tests/test_plane_runner.py @@ -1,9 +1,12 @@ -"""ax runner contract (ai-service-plane task 3, Requirements 4.1-4.4): env contract, readiness, delivery, exit codes.""" +"""ax runner contract (ai-service-plane task 3, Requirements 4.1-4.6): env contract, readiness, delivery, exit codes, +the start request and credentials.""" from __future__ import annotations +import hashlib import io import json +import logging import os import shutil import signal @@ -23,9 +26,13 @@ import pytest from openultrasast.plane import runner from openultrasast.plane.manifests import GitSource +from openultrasast.plane.router import Router, StartError from openultrasast.plane.runner import ( EXIT_BY_STATUS, + START_PATH, DeliveryError, + Start, + StartGate, clone_destination, deliver, find_cached_checkout, @@ -61,6 +68,7 @@ def env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, stub_root: Path) -> Ite monkeypatch.delenv(key) monkeypatch.setenv("AX_RUNNER_HTTP", "0") monkeypatch.setenv("AX_RUNNER_EXIT_AFTER_COMMAND", "1") + monkeypatch.setenv("AX_RUNNER_AUTOSTART", "1") # the start-request tests below take it away again monkeypatch.setenv("AX_RUNNER_TASK_PACKAGE", STUB_PACKAGE) monkeypatch.setenv("OUSAST_STATE_DIR", str(tmp_path / "state")) monkeypatch.setenv("OUSAST_DELIVERY_BACKOFF", "0") @@ -341,12 +349,14 @@ def wait_for(predicate: object, timeout: float = 30.0) -> None: time.sleep(0.05) -def spawn_runner(tmp_path: Path, port: int, **extra: str) -> subprocess.Popen[bytes]: - """``python -m openultrasast.plane.runner`` as the image runs it: HTTP on, no exit-after-command escape.""" - child_env = {k: v for k, v in os.environ.items() if k != "AX_RUNNER_EXIT_AFTER_COMMAND"} +def spawn_runner(tmp_path: Path, port: int, *, autostart: bool = True, log_name: str = "runner.log", **extra: str) -> subprocess.Popen: + """``python -m openultrasast.plane.runner`` as the image runs it: HTTP on, no exit-after-command escape; + ``autostart=False`` also drops the autostart escape, so the command waits for a start request.""" + dropped = {"AX_RUNNER_EXIT_AFTER_COMMAND"} | (set() if autostart else {"AX_RUNNER_AUTOSTART"}) + child_env = {k: v for k, v in os.environ.items() if k not in dropped} child_env.update(AX_TASK_YAML=task_yaml(tmp_path / "out", tmp_path / "ws"), AX_WORKSPACES_YAML=files_workspace_yaml()) child_env.update(AX_RUNNER_HTTP="1", AX_RUNNER_PORT=str(port), **extra) - log = (tmp_path / "runner.log").open("wb") + log = (tmp_path / log_name).open("wb") return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log) @@ -609,3 +619,138 @@ def test_the_runner_accepts_what_ax_hands_it() -> None: assert list(spaces) == ["w"] with pytest.raises(RunnerError, match="uid"): load_task("apiVersion: ax.io/v1alpha1\nkind: Task\nmetadata:\n name: t\n uid: x\nspec: {}\n") + + +# --- the start request and credentials (Requirements 4.5, 4.6) ----------------------------------------------------------- + +SECRET = "sk-test-0f3c9a-never-written-anywhere" +SECRET_SHA = hashlib.sha256(SECRET.encode()).hexdigest() + + +def post_start(port: int | str, body: object) -> tuple[int, str]: + data = json.dumps(body).encode() + request = urllib.request.Request(f"http://127.0.0.1:{port}{START_PATH}", data=data, method="POST") + try: + with urllib.request.urlopen(request, timeout=5) as response: + return int(response.status), response.read().decode() + except urllib.error.HTTPError as exc: + return exc.code, exc.read().decode() + + +def ready_on(port: int | str) -> bool: + try: + return get(f"http://127.0.0.1:{port}/readyz") == 200 + except urllib.error.URLError: + return False # not listening yet + + +def files_containing(root: Path, needle: str) -> list[str]: + return sorted(str(p) for p in root.rglob("*") if p.is_file() and needle.encode() in p.read_bytes()) + + +def main_in_thread(tmp_path: Path, **extra: str) -> tuple[threading.Thread, dict[str, str], list[int]]: + """``runner.main`` on a private environment in a thread: HTTP on a free port, no autostart; the exit code lands + in the returned list once the command was started and delivered.""" + environ = {k: v for k, v in os.environ.items() if k != "AX_RUNNER_AUTOSTART"} + environ.update(AX_TASK_YAML=task_yaml(tmp_path / "out", tmp_path / "ws"), AX_WORKSPACES_YAML=files_workspace_yaml()) + environ.update(AX_RUNNER_HTTP="1", AX_RUNNER_PORT="0", OUSAST_RUN="run-1", STUB_SECRET_VAR="DEEPSEEK_API_KEY", **extra) + codes: list[int] = [] + thread = threading.Thread(target=lambda: codes.append(runner.main(environ=environ)), daemon=True) + thread.start() + wait_for(lambda: "AX_RUNNER_BOUND_PORT" in environ) + wait_for(lambda: ready_on(environ["AX_RUNNER_BOUND_PORT"])) + return thread, environ, codes + + +def test_the_command_waits_for_the_start_request_and_the_credential_reaches_only_the_child( + env: dict[str, str], receiver: Receiver, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level(logging.DEBUG) + counter = tmp_path / "runs.txt" + thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter)) + port = environ["AX_RUNNER_BOUND_PORT"] + time.sleep(0.5) + assert not counter.exists() and thread.is_alive(), "ready, but the command waits for the start request" + assert post_start(port, {"run": "run-2", "task": "stub"})[0] == 409, "another run's start" + assert post_start(port, {"run": "run-1", "task": "golden"})[0] == 409, "another task's start" + assert post_start(port, {"run": "run-1", "task": "stub", "credentials": {"lower": "x"}})[0] == 400 + assert post_start(port, {"run": "run-1", "task": "stub", "credentials": {"OUSAST_OUTPUT_DIR": "/"}})[0] == 400 + assert not counter.exists() + status, text = post_start(port, {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}}) + assert (status, text) == (202, "started") + status, text = post_start(port, {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}}) + assert status == 409 and "already started" in text, "one start per boot" + thread.join(timeout=30) + assert codes == [0] and len(counter.read_text().splitlines()) == 1 + facts = json.loads((tmp_path / "out" / "facts.json").read_text()) + assert facts["secret_sha256"] == SECRET_SHA, "the credential is in the command's environment" + assert "DEEPSEEK_API_KEY" not in environ and "DEEPSEEK_API_KEY" not in os.environ, "and only there" + assert str(tmp_path / "out" / "facts.json") in files_containing(tmp_path, SECRET_SHA), "the scan reads the output files" + assert files_containing(tmp_path, SECRET) == [], "no file holds the credential" + assert SECRET not in receiver.posts[0][1].decode("latin-1"), "nor the delivered tar" + assert all(SECRET not in record.getMessage() for record in caplog.records), "nor any log record" + assert any("start request: 202" in record.getMessage() for record in caplog.records) + + +def test_a_crash_that_prints_the_credential_is_redacted( + env: dict[str, str], receiver: Receiver, tmp_path: Path, capfd: pytest.CaptureFixture[str] +) -> None: + thread, environ, codes = main_in_thread(tmp_path, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="leak") + start = {"run": "run-1", "task": "stub", "credentials": {"DEEPSEEK_API_KEY": SECRET}} + assert post_start(environ["AX_RUNNER_BOUND_PORT"], start)[0] == 202 + thread.join(timeout=30) + assert codes == [EXIT_BY_STATUS["failed"]] + summary = json.loads((tmp_path / "out" / "summary.json").read_text()) + assert "auth failed for key [redacted]" in summary["reason"], "the stderr tail keeps the line, not the value" + assert files_containing(tmp_path, SECRET) == [] + assert SECRET not in capfd.readouterr().err, "the echoed stderr is redacted too" + + +def test_start_before_the_workspaces_are_ready_is_503_and_a_delivered_run_refuses_it() -> None: + gate = StartGate() + assert gate.offer(b'{"run": "r", "task": "t"}')[0] == 503, "the reconciler retries until the workspaces are ready" + gate.refuse("task t already ran and was delivered (done)") + status, text = gate.offer(b'{"run": "r", "task": "t"}') + assert status == 409 and "already ran and was delivered" in text + assert "sk-" not in repr(Start({"K": "sk-x"})), "a Start never prints a value" + + +def test_http_off_without_autostart_is_refused(env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.delenv("AX_RUNNER_AUTOSTART") + code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml()) + assert code == EXIT_BY_STATUS["failed"] and not (output / "facts.json").exists() + + +def test_the_golden_boot_never_runs_the_command(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None: + """Agent Substrate's golden-snapshot boot runs the same image with the same env; only the router-addressed + actor gets the start request, so only it runs the command. A later boot of a delivered task refuses a start.""" + counter = tmp_path / "runs.txt" + same = {"OUSAST_ARTIFACT_URL": receiver.url, "STUB_COUNTER": str(counter), "OUSAST_RUN": "run-1"} + real_port, golden_port = free_port(), free_port() + real = spawn_runner(tmp_path, real_port, autostart=False, log_name="real.log", **same) + golden = spawn_runner(tmp_path, golden_port, autostart=False, log_name="golden.log", **same) + try: + for port in (real_port, golden_port): + wait_for(lambda port=port: ready_on(port)) + Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {"DEEPSEEK_API_KEY": SECRET}) + marker = tmp_path / "state" / "stub.done.json" + wait_for(marker.exists) + time.sleep(1.0) + assert len(counter.read_text().splitlines()) == 1, "the golden boot never ran the command" + assert golden.poll() is None and len(receiver.posts) == 1 + with pytest.raises(StartError, match="already started"): + Router(f"http://127.0.0.1:{real_port}").start_task("default", "stub", "run-1", {}) + again_port = free_port() + again = spawn_runner(tmp_path, again_port, autostart=False, log_name="again.log", **same) + try: + with pytest.raises(StartError, match="already ran and was delivered"): + Router(f"http://127.0.0.1:{again_port}").start_task("default", "stub", "run-1", {}) + finally: + again.kill() + again.wait() + finally: + for proc in (real, golden): + proc.kill() + proc.wait() + assert len(counter.read_text().splitlines()) == 1 + assert files_containing(tmp_path, SECRET) == [], "no log, workspace, output or state file holds the credential"