fix(plane): ax task lifecycle -- resume, stay-up runner, delivery as completion

The first live run on the local ax cluster and ax's source (docs/runner.md,
docs/sandbox.md, internal/controller/reconciler.go, internal/server/server.go)
showed a lifecycle the task-3/4 code did not model:

- `ax apply` creates a Task Suspended (Ready=False, TaskSuspended); it runs only
  after `ax resume task`. The first resume of a new image can fail with
  DeadlineExceeded while Agent Substrate builds the golden snapshot, and ax's
  server then stores phase Failed; a later resume succeeds.
- ax has no Completed phase and never reads a command's exit back.
- The runner is PID 1 and must keep serving after the command exits; stop and
  suspend deliver SIGTERM, to be forwarded to the command's process group.
  Our runner imported the task in-process and exited, which stopped the
  container ("Container guest is stopped") and made every resume time out.
- ax's controller probes /readyz?check=workspace; the query string made our
  handler answer 404.

Runner (Req 4.1-4.4): spec.command runs as a child
`python -m openultrasast.plane.tasks.<name>` in its own process group, cwd the
first workspace, AX_METADATA_URL set; ax's metadata paths are served. After the
child exits the runner reads summary.json (a crash without one becomes failed
with the exit code and stderr tail), delivers the output directory, writes
/workspace/.ousast-state/<task>.done.json and serves until SIGTERM (forwarded
to the group, SIGKILL after OUSAST_TERM_GRACE, exit 0). A boot with the marker
never reruns the command; a failed delivery is retried alone. Workspaces are
prepared on the first boot only. AX_RUNNER_EXIT_AFTER_COMMAND=1 is the
unit-test escape.

Reconciler (Req 2.2, 2.3, 3.1-3.3): `ax resume task` after apply, retrying
DeadlineExceeded/Unavailable with backoff up to OUSAST_RESUME_TIMEOUT (900 s);
completion is the artifact delivery; Failed after a resume went through, a
vanished task, or no delivery within OUSAST_TASK_TIMEOUT (7200 s) fail the
task with ax's condition message; `ax delete task` follows. Ax gains resume()
and phase() returns the condition message. doctor moves to plane/doctor.py
(tasks.md task 0 allows the sibling) to keep reconciler.py under 500 lines.

Tests: the fake ax follows the real lifecycle (Suspended, a failing first
resume that leaves Failed, Running forever, delivery only after a resume went
through), plus Failed-with-message, vanish, resume-timeout and task-timeout
scenarios; runner tests cover the child's cwd/group/metadata URL, stay-up and
SIGTERM in a real subprocess, SIGKILL of a SIGTERM-ignoring group, and the
marker. The opt-in live test now checks resume, Running, suspend and a second
resume instead of a Completed phase. design.md and tasks.md state the
lifecycle; the module audit manifest counts plane.doctor.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

tests/plane_stub_task.py

@@ -1,34 +1,57 @@
 """A stub task module for the runner tests: records its contract, writes one artifact and ``summary.json``.
 
-Registered by the tests as ``openultrasast.plane.tasks.stub_task``; ``STUB_MODE`` picks the outcome
-(``done``, ``failed``, ``unfinished``, ``crash``, ``nosummary``, ``exit``).
+The tests copy it into a throwaway package (``AX_RUNNER_TASK_PACKAGE``) so the runner starts it the way it starts
+a real task, ``python -m <package>.stub_task``. ``STUB_MODE`` picks the outcome (``done``, ``failed``,
+``unfinished``, ``crash``, ``nosummary``, ``exit``, ``sleep``); ``STUB_COUNTER`` names a file that gets one line
+per run.
 """
 
 from __future__ import annotations
 
 import json
 import os
+import signal
+import subprocess
+import sys
+import time
 import urllib.request
 from pathlib import Path
 
 
+def _get(url: str) -> tuple[int, str]:
+    with urllib.request.urlopen(url, timeout=5) as response:
+        return int(response.status), response.read().decode("utf-8")
+
+
 def main(argv: list[str]) -> None:
     out = Path(os.environ["OUSAST_OUTPUT_DIR"])
     out.mkdir(parents=True, exist_ok=True)
     mode = os.environ.get("STUB_MODE", "done")
+    counter = os.environ.get("STUB_COUNTER")
+    if counter:
+        with open(counter, "a", encoding="utf-8") as handle:
+            handle.write(f"{os.getpid()}\n")
     workspace = Path(os.environ["OUSAST_WORKSPACE_DIR"])
-    readyz: int | None = None
     port = os.environ.get("AX_RUNNER_BOUND_PORT")
-    if port:
-        with urllib.request.urlopen(f"http://127.0.0.1:{port}/readyz", timeout=5) as response:
-            readyz = int(response.status)
+    readyz = _get(f"http://127.0.0.1:{port}/readyz")[0] if port else None
+    metadata = os.environ.get("AX_METADATA_URL")
     record = {
         "argv": argv,
         "env": {k: v for k, v in os.environ.items() if k.startswith("OUSAST_")},
         "workspace_files": sorted(p.relative_to(workspace).as_posix() for p in workspace.rglob("*") if p.is_file()),
         "readyz": readyz,
+        "cwd": os.getcwd(),
+        "pid": os.getpid(),
+        "pgid": os.getpgid(0),
+        "metadata_url": metadata,
+        "metadata_task": _get(f"{metadata}/metadata/v1alpha1/ax/task")[1] if metadata else None,
     }
     (out / "facts.json").write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8")
+    if mode == "sleep":  # ignore SIGTERM so only the runner's SIGKILL ends the group; the grandchild inherits it
+        signal.signal(signal.SIGTERM, signal.SIG_IGN)
+        grandchild = subprocess.Popen(["sleep", "60"])
+        (out / "pids.json").write_text(json.dumps([os.getpid(), grandchild.pid]), encoding="utf-8")
+        time.sleep(60)
     if mode == "crash":
         raise RuntimeError("stub exploded")
     if mode == "exit":
@@ -37,3 +60,7 @@ def main(argv: list[str]) -> None:
         return
     summary = {"status": mode, "units_done": 1, "units_total": 1, "usd": 0.0, "calls": 0, "usage": {}}
     (out / "summary.json").write_text(json.dumps(summary), encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

tests/test_plane_ax_live.py

@@ -8,12 +8,13 @@ passes, so the host suite never depends on cluster state. It applies the digest-
 from __future__ import annotations
 
 import os
+import subprocess
 import time
 from pathlib import Path
 
 import pytest
 
-from openultrasast.plane.reconciler import Ax, doctor
+from openultrasast.plane.reconciler import TRANSIENT, Ax, doctor
 
 pytestmark = pytest.mark.ax
 SMOKE = Path.home() / ".cache/ousast/ax-src/smoke-task.yaml"
@@ -32,16 +33,34 @@ def live_ax() -> Ax:
     return Ax()
 
 
-def test_smoke_task_completes_on_the_cluster(live_ax: Ax) -> None:
+def resume_until_accepted(ax: Ax, name: str, deadline: float) -> None:
+    """ax creates a Task Suspended; the first resume of a new image may time out while its snapshot is built."""
+    error: str | None = "not tried"
+    while error is not None and time.monotonic() < deadline:
+        error = ax.resume(name)
+        if error is not None and not TRANSIENT.search(error):
+            raise AssertionError(f"ax resume task {name} failed: {error}")
+        if error is not None:
+            time.sleep(10)
+    assert error is None, f"ax resume task {name} never went through: {error}"
+
+
+def test_smoke_task_runs_and_its_runner_survives_suspend_and_resume(live_ax: Ax) -> None:
+    """ax has no Completed phase: the smoke task must reach Running after resume, and its runner must stay up after
+    the command so that a suspend and a second resume still go through (an exiting runner made resumes time out)."""
+    name = "ousast-smoke"
+    live_ax.delete(name)
     live_ax.apply(SMOKE)
-    deadline = time.monotonic() + 600
-    phase = "Pending"
+    deadline = time.monotonic() + 900
     try:
-        while time.monotonic() < deadline:
-            phase = live_ax.phase("ousast-smoke")
-            if any(word in phase.lower() for word in ("completed", "succeeded", "failed", "error")):
-                break
-            time.sleep(5)
+        resume_until_accepted(live_ax, name, deadline)
+        phase, message = live_ax.phase(name)
+        assert phase == "Running", f"after resume: {phase} ({message})"
+        time.sleep(30)  # the model-free repo-facts command is finished by now
+        suspended = subprocess.run(["ax", "suspend", "task", name], capture_output=True, text=True, check=False)
+        assert suspended.returncode == 0, suspended.stderr
+        resume_until_accepted(live_ax, name, deadline)
+        phase, message = live_ax.phase(name)
+        assert phase == "Running", f"after the second resume: {phase} ({message})"
     finally:
-        live_ax.delete("ousast-smoke")
-    assert phase.lower() in ("completed", "succeeded"), f"smoke task ended in phase {phase!r}"
+        live_ax.delete(name)

tests/test_plane_reconciler.py

@@ -1,9 +1,11 @@
 """ai-service-plane Req 3.1-3.5, 2.2, 7.3: a fake ``ax`` drives a Run through the reconciler.
 
-The fake is a Python script the tests write into ``tmp_path``: it logs every argv line, parses what ``apply``
-receives, answers ``get task`` with Running until a scripted poll count and then a terminal phase, and on reaching
-it POSTs a scripted tar to the ``OUSAST_ARTIFACT_URL`` it found in the applied Task's env. Nothing here runs a
-task; every outcome comes from a scripted ``summary.json`` delivery.
+The fake is a Python script the tests write into ``tmp_path`` that follows ax's real task lifecycle (ax source:
+internal/controller/reconciler.go, internal/server/server.go): ``apply`` creates the task ``Suspended``; ``resume``
+fails with DeadlineExceeded the first time (ax then reports ``Failed``, as while Agent Substrate builds a new
+image's golden snapshot) and succeeds after; ``get task`` answers ``Running`` forever -- ax has no Completed phase
+-- and on the scripted poll it POSTs a scripted tar to the ``OUSAST_ARTIFACT_URL`` in the applied Task's env, or
+turns ``Failed`` with a condition message, or vanishes. Every outcome comes from a scripted delivery.
 """
 
 from __future__ import annotations
@@ -60,7 +62,8 @@ def non_ax_fields(value: object, schema: dict | None, path: str) -> list[str]:
 
 
 FAKE_AX = '''#!{python}
-"""Fake ax CLI for the reconciler tests: argv log, scripted phases, scripted deliveries."""
+"""Fake ax CLI with ax's real task lifecycle: apply -> Suspended; resume (the first ones fail DeadlineExceeded and
+leave the task Failed, as ax's server does) -> Running; never Completed; the scripted delivery after N polls."""
 import io, json, sys, tarfile, time, urllib.error, urllib.request
 from pathlib import Path
 import yaml
@@ -70,42 +73,73 @@ SCRIPT = json.loads((HERE / "script.json").read_text())
 argv = sys.argv[1:]
 (HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} " + " ".join(argv) + "\\n")
 (HERE / "applied").mkdir(exist_ok=True)
+record = HERE / "applied" / ((argv[2] if len(argv) > 2 else "-") + ".json")
+rec = json.loads(record.read_text()) if record.exists() else None
+script = SCRIPT.get(rec["env"]["OUSAST_TASK"], {{}}) if rec else {{}}
+
+
+def gone():
+    print(f'Error: getting task "{{argv[2]}}": rpc error: code = NotFound desc = task not found', file=sys.stderr)
+    sys.exit(1)
+
+
 if argv[:2] == ["apply", "-f"]:
     docs = list(yaml.safe_load_all(Path(argv[2]).read_text()))
     for doc in docs:
         if doc["kind"] == "Task":
             env = {{e["name"]: e["value"] for e in doc["spec"].get("env", [])}}
-            (HERE / "applied" / (doc["metadata"]["name"] + ".json")).write_text(json.dumps({{"docs": docs, "env": env, "polls": 0}}))
+            state = {{"docs": docs, "env": env, "polls": 0, "resumes": 0, "phase": "Suspended", "message": "Task is suspended"}}
+            (HERE / "applied" / (doc["metadata"]["name"] + ".json")).write_text(json.dumps(state))
     print("task created")
-elif argv[:2] == ["get", "task"]:
-    record = HERE / "applied" / (argv[2] + ".json")
-    if not record.exists():
-        print("Error: not found", file=sys.stderr)
+elif argv[:2] == ["resume", "task"]:
+    if rec is None or rec.get("gone"):
+        gone()
+    rec["resumes"] += 1
+    if rec["resumes"] <= script.get("resume_failures", 1):
+        rec.update(phase="Failed", message="resuming actor: context deadline exceeded")
+        record.write_text(json.dumps(rec))
+        print("Error: resuming task: rpc error: code = DeadlineExceeded desc = context deadline exceeded", file=sys.stderr)
         sys.exit(1)
-    rec = json.loads(record.read_text())
-    rec["polls"] += 1
-    env, task = rec["env"], rec["env"]["OUSAST_TASK"]
-    script = SCRIPT.get(task, {{}})
-    needed = script.get("polls", 1)
-    phase = "Running" if rec["polls"] < needed else script.get("phase", "Completed")
-    if rec["polls"] == needed and script.get("deliver", True):
-        buf = io.BytesIO()
-        with tarfile.open(fileobj=buf, mode="w") as tar:
-            for path, content in script.get("files", {{}}).items():
-                data = (content if isinstance(content, str) else json.dumps(content)).encode()
-                info = tarfile.TarInfo(path)
-                info.size = len(data)
-                tar.addfile(info, io.BytesIO(data))
-        headers = {{"Content-Type": "application/x-tar", "X-Ousast-Task": script.get("as_task", task), "X-Ousast-Run": env["OUSAST_RUN"]}}
-        req = urllib.request.Request(env["OUSAST_ARTIFACT_URL"], data=buf.getvalue(), method="POST", headers=headers)
-        try:
-            with urllib.request.urlopen(req) as resp:
-                (HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} delivered {{task}} {{resp.status}}\\n")
-        except urllib.error.HTTPError as exc:
-            (HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} rejected {{task}} {{exc.code}}\\n")
+    rec.update(phase="Running", message="Task is running and its workspace is ready")
     record.write_text(json.dumps(rec))
-    print(yaml.safe_dump({{"apiVersion": "ax.io/v1alpha1", "kind": "Task", "metadata": {{"name": argv[2]}}, "status": {{"phase": phase}}}}))
+    print(f"task.ax.io/{{argv[2]}} resumed")
+elif argv[:2] == ["get", "task"]:
+    if rec is None or rec.get("gone"):
+        gone()
+    if rec["phase"] == "Running":
+        rec["polls"] += 1
+    env, task = rec["env"], rec["env"]["OUSAST_TASK"]
+    if rec["phase"] == "Running" and rec["polls"] == script.get("polls", 1):
+        if "phase" in script:
+            rec.update(phase=script["phase"], message=script.get("message", ""))
+        if script.get("vanish"):
+            rec["gone"] = True
+        if script.get("deliver", True) and "phase" not in script and not script.get("vanish"):
+            buf = io.BytesIO()
+            with tarfile.open(fileobj=buf, mode="w") as tar:
+                for path, content in script.get("files", {{}}).items():
+                    data = (content if isinstance(content, str) else json.dumps(content)).encode()
+                    info = tarfile.TarInfo(path)
+                    info.size = len(data)
+                    tar.addfile(info, io.BytesIO(data))
+            headers = {{"Content-Type": "application/x-tar", "X-Ousast-Run": env["OUSAST_RUN"]}}
+            headers["X-Ousast-Task"] = script.get("as_task", task)
+            req = urllib.request.Request(env["OUSAST_ARTIFACT_URL"], data=buf.getvalue(), method="POST", headers=headers)
+            try:
+                with urllib.request.urlopen(req) as resp:
+                    (HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} delivered {{task}} {{resp.status}}\\n")
+            except urllib.error.HTTPError as exc:
+                (HERE / "ax.log").open("a").write(f"{{time.monotonic():.4f}} rejected {{task}} {{exc.code}}\\n")
+    record.write_text(json.dumps(rec))
+    if rec.get("gone"):
+        gone()
+    condition = {{"type": "Ready", "status": "True" if rec["phase"] == "Running" else "False", "message": rec["message"]}}
+    status = {{"phase": rec["phase"], "conditions": [condition]}}
+    print(yaml.safe_dump({{"apiVersion": "ax.io/v1alpha1", "kind": "Task", "metadata": {{"name": argv[2]}}, "status": status}}))
 elif argv[:2] == ["delete", "task"]:
+    if rec is not None:
+        rec["gone"] = True
+        record.write_text(json.dumps(rec))
     print("deleted")
 '''
 
@@ -222,7 +256,7 @@ class Fake:
         for stamp, words in self.log():
             if verb == "apply" and words[:2] == ["apply", "-f"]:
                 out.append((stamp, Path(words[2]).parent.name))
-            elif verb == "delete" and words[:2] == ["delete", "task"]:
+            elif verb in ("delete", "resume") and words[:2] == [verb, "task"]:
                 out.append((stamp, words[2]))
             elif verb in ("delivered", "rejected") and words[0] == verb:
                 out.append((stamp, words[1]))
@@ -234,6 +268,7 @@ def fake(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Fake:
     monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path / "results"))
     monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
     monkeypatch.setenv("OUSAST_POLL_SECONDS", "0.02")
+    monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "20")  # a test that never delivers fails in seconds, not in 2 hours
     fake = Fake(tmp_path)
     fake.script(SCRIPT)
     return fake
@@ -343,18 +378,57 @@ def test_failed_task_halts_the_run(fake: Fake, tmp_path: Path) -> None:
     assert [name for _, name in fake.events("apply")] == ["repo-facts"]
 
 
-def test_ax_failure_phase_and_missing_delivery_are_failed(fake: Fake, tmp_path: Path) -> None:
+def test_resume_retries_deadline_exceeded_and_completion_is_the_delivery(fake: Fake, tmp_path: Path) -> None:
+    """ax creates a Task Suspended; the first resume of a new image times out; a later one runs it (live evidence)."""
+    assert reconciler.run(write_run(tmp_path, "life", "    - name: repo-facts\n      task: repo-facts\n"), ax=str(fake.ax)) == "done"
+    verbs = [w[0] for _, w in fake.log() if w[0] in ("apply", "resume", "delivered", "delete")]
+    assert verbs == ["apply", "resume", "resume", "delivered", "delete"], "resumed twice, delivered, then deleted"
+    resumes = [stamp for stamp, _ in fake.events("resume")]
+    (delivered, _), (deleted, _) = fake.events("delivered")[0], fake.events("delete")[0]
+    assert resumes[1] < delivered < deleted, "the fake delivers only once a resume went through"
+    assert state_of("life")["tasks"]["repo-facts"]["status"] == "done", "done without any Completed phase from ax"
+
+
+def test_resume_gives_up_after_the_resume_timeout_with_the_last_error(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("OUSAST_RESUME_TIMEOUT", "0.3")
+    fake.script({**SCRIPT, "repo-facts": {**SCRIPT["repo-facts"], "resume_failures": 1000}})
+    assert reconciler.run(write_run(tmp_path, "stuck"), ax=str(fake.ax)) == "failed"
+    reason = state_of("stuck")["tasks"]["repo-facts"]["reason"]
+    assert "OUSAST_RESUME_TIMEOUT" in reason and "DeadlineExceeded" in reason
+    assert len(fake.events("resume")) >= 2, "retried with backoff before giving up"
+    assert [n for _, n in fake.events("delete")] == ["stuck-repo-facts"]
+
+
+def test_ax_failed_phase_before_delivery_fails_with_the_condition_message(fake: Fake, tmp_path: Path) -> None:
     script = json.loads(json.dumps(SCRIPT))
-    script["repo-facts"]["phase"] = "Failed"
+    script["repo-facts"].update(phase="Failed", message="ActorResumeFailed: worker lost")
     fake.script(script)
     assert reconciler.run(write_run(tmp_path, "phase"), ax=str(fake.ax)) == "failed"
-    assert state_of("phase")["tasks"]["repo-facts"]["reason"] == "ax phase Failed"
-    script = json.loads(json.dumps(SCRIPT))
-    script["repo-facts"]["deliver"] = False
-    fake.script(script)
+    assert state_of("phase")["tasks"]["repo-facts"]["reason"] == "ax phase Failed: ActorResumeFailed: worker lost"
+    assert statuses("phase") == {"repo-facts": "failed", "verify": "pending", "agree": "pending"}
+    assert [n for _, n in fake.events("delete")] == ["phase-repo-facts"], "ax delete follows a failed task"
+
+
+def test_a_task_that_vanishes_before_delivery_is_failed(fake: Fake, tmp_path: Path) -> None:
+    fake.script({**SCRIPT, "repo-facts": {**SCRIPT["repo-facts"], "vanish": True}})
+    assert reconciler.run(write_run(tmp_path, "vanish"), ax=str(fake.ax)) == "failed"
+    assert "disappeared from ax" in state_of("vanish")["tasks"]["repo-facts"]["reason"]
+
+
+def test_no_delivery_within_the_task_timeout_is_failed(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "0.5")
+    fake.script({**SCRIPT, "repo-facts": {**SCRIPT["repo-facts"], "deliver": False}})
+    started = time.monotonic()
     assert reconciler.run(write_run(tmp_path, "silent"), ax=str(fake.ax)) == "failed"
-    assert state_of("silent")["tasks"]["repo-facts"]["reason"] == "no summary.json delivered"
-    assert [n for _, n in fake.events("delete")] == ["phase-repo-facts", "silent-repo-facts"], "ax delete follows every terminal task"
+    assert time.monotonic() - started >= 0.5
+    reason = state_of("silent")["tasks"]["repo-facts"]["reason"]
+    assert "OUSAST_TASK_TIMEOUT" in reason and "Running" in reason, "Running forever is not completion"
+    assert [n for _, n in fake.events("delete")] == ["silent-repo-facts"], "ax delete follows every task"
+
+
+def test_outcome_of_needs_a_delivered_summary() -> None:
+    assert reconciler.outcome_of(None) == ("failed", "no summary.json delivered")
+    assert reconciler.outcome_of({"status": "done", "units_done": 1, "units_total": 1}) == ("done", "")
 
 
 def test_summary_done_requires_every_unit(fake: Fake, tmp_path: Path) -> None:
@@ -393,7 +467,8 @@ def test_receiver_rejects_traversal_unknown_tasks_and_wrong_types(tmp_path: Path
     receiver.server_close()
 
 
-def test_delivery_for_a_task_that_is_not_running_is_rejected(fake: Fake, tmp_path: Path) -> None:
+def test_delivery_for_a_task_that_is_not_running_is_rejected(fake: Fake, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("OUSAST_TASK_TIMEOUT", "1")  # the rejected delivery is no delivery: the task times out
     script = json.loads(json.dumps(SCRIPT))
     script["repo-facts"]["as_task"] = "agree"  # the runner claims to be a task that has not started
     fake.script(script)
@@ -544,12 +619,12 @@ def test_elapsed_polling_is_paced(fake: Fake, tmp_path: Path, monkeypatch: pytes
 
 def test_doctor_treats_an_empty_namespace_as_not_ready(monkeypatch: pytest.MonkeyPatch) -> None:
     """kubectl's "No resources found" must not read as one ready pod (a plausible-zero instrument failure)."""
-    from openultrasast.plane import reconciler
+    from openultrasast.plane import doctor
 
-    monkeypatch.setattr(reconciler, "_sh", lambda *a: (True, "No resources found in ax-system namespace."))
-    ok, text = reconciler._pods_ready("ax-system", "kind-ousast")
+    monkeypatch.setattr(doctor, "_sh", lambda *a: (True, "No resources found in ax-system namespace."))
+    ok, text = doctor._pods_ready("ax-system", "kind-ousast")
     assert not ok and "no pods" in text
-    monkeypatch.setattr(reconciler, "_sh", lambda *a: (True, "a-1 1/1 Running 0 1m\nb-2 0/1 Completed 0 1m"))
-    assert reconciler._pods_ready("ns", "ctx") == (True, "2 pods ready")
-    monkeypatch.setattr(reconciler, "_sh", lambda *a: (True, "a-1 0/1 Running 0 1m"))
-    assert reconciler._pods_ready("ns", "ctx")[0] is False
+    monkeypatch.setattr(doctor, "_sh", lambda *a: (True, "a-1 1/1 Running 0 1m\nb-2 0/1 Completed 0 1m"))
+    assert doctor._pods_ready("ns", "ctx") == (True, "2 pods ready")
+    monkeypatch.setattr(doctor, "_sh", lambda *a: (True, "a-1 0/1 Running 0 1m"))
+    assert doctor._pods_ready("ns", "ctx")[0] is False

tests/test_plane_runner.py

@@ -2,14 +2,17 @@
 
 from __future__ import annotations
 
-import importlib.util
 import io
 import json
 import os
+import shutil
+import signal
+import socket
 import subprocess
 import sys
 import tarfile
 import threading
+import time
 import urllib.error
 import urllib.request
 from collections.abc import Iterator
@@ -30,36 +33,38 @@ from openultrasast.plane.runner import (
 )
 
 STUB = Path(__file__).parent / "plane_stub_task.py"
-STUB_MODULE = f"{runner.TASK_PACKAGE}.stub_task"
+STUB_PACKAGE = "ousast_stub_tasks"
+SRC = Path(runner.__file__).resolve().parents[2]  # the ``src`` this test imports; the children must import it too
 
 
-@pytest.fixture(autouse=True)
-def stub_task() -> Iterator[None]:
-    """Register the stub as ``openultrasast.plane.tasks.stub_task`` for the runner's import; no src package needed."""
-    spec = importlib.util.spec_from_file_location(STUB_MODULE, STUB)
-    assert spec is not None and spec.loader is not None
-    module = importlib.util.module_from_spec(spec)
-    sys.modules[STUB_MODULE] = module
-    spec.loader.exec_module(module)
-    try:
-        yield
-    finally:
-        sys.modules.pop(STUB_MODULE, None)
+@pytest.fixture(scope="session")
+def stub_root(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    """A throwaway package holding the stub, so the runner starts it as ``python -m ousast_stub_tasks.stub_task``."""
+    root = tmp_path_factory.mktemp("stubs")
+    (root / STUB_PACKAGE).mkdir()
+    (root / STUB_PACKAGE / "__init__.py").write_text("", encoding="utf-8")
+    shutil.copy(STUB, root / STUB_PACKAGE / "stub_task.py")
+    return root
 
 
 @pytest.fixture
-def env(monkeypatch: pytest.MonkeyPatch) -> Iterator[dict[str, str]]:
+def env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, stub_root: Path) -> Iterator[dict[str, str]]:
     """A private environment for one runner invocation, restored whole afterwards.
 
     The runner and the stub read and write ``os.environ`` directly (that is the contract), so monkeypatch alone
-    would not undo what the runner exports; the snapshot does.
+    would not undo what the runner exports; the snapshot does. ``AX_RUNNER_EXIT_AFTER_COMMAND`` makes ``main()``
+    return after delivery instead of serving until SIGTERM, as the image does.
     """
     saved = dict(os.environ)
     for key in list(os.environ):
         if key.startswith(("OUSAST_", "AX_", "STUB_")):
             monkeypatch.delenv(key)
     monkeypatch.setenv("AX_RUNNER_HTTP", "0")
+    monkeypatch.setenv("AX_RUNNER_EXIT_AFTER_COMMAND", "1")
+    monkeypatch.setenv("AX_RUNNER_TASK_PACKAGE", STUB_PACKAGE)
+    monkeypatch.setenv("OUSAST_STATE_DIR", str(tmp_path / "state"))
     monkeypatch.setenv("OUSAST_DELIVERY_BACKOFF", "0")
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(filter(None, [str(stub_root), str(SRC), os.environ.get("PYTHONPATH")])))
     try:
         yield os.environ
     finally:
@@ -156,8 +161,10 @@ def test_readyz_is_503_until_workspaces_are_ready_then_200() -> None:
         assert get(f"{base}/healthz") == 200
         assert get(f"{base}/readyz") == 503
         assert get(f"{base}/other") == 404
+        assert get(f"{base}/readyz?check=workspace") == 503, "ax's controller probes with a query string"
         ready.set()
         assert get(f"{base}/readyz") == 200
+        assert get(f"{base}/readyz?check=workspace") == 200
     finally:
         server.stop()
 
@@ -297,6 +304,128 @@ def test_an_unknown_task_module_is_failed(env: dict[str, str], monkeypatch: pyte
     assert "no_such_task" in json.loads((output / "summary.json").read_text())["reason"]
 
 
+# --- ax's lifecycle: a child process, stay up, SIGTERM, completion marker (docs/runner.md, docs/sandbox.md) -----------------
+
+
+def test_the_command_is_a_child_in_its_own_group_in_the_first_workspace(
+    env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), AX_RUNNER_HTTP="1", AX_RUNNER_PORT="0")
+    assert code == 0
+    facts = json.loads((output / "facts.json").read_text())
+    assert facts["pid"] != os.getpid(), "the task is a child process, not an import"
+    assert facts["pgid"] == facts["pid"] != os.getpgid(0), "the child leads its own process group"
+    assert facts["cwd"] == str(tmp_path / "ws"), "the first workspace is the working directory"
+    assert facts["metadata_url"] == f"http://127.0.0.1:{os.environ['AX_RUNNER_BOUND_PORT']}"
+    assert "name: stub" in facts["metadata_task"], "AX_METADATA_URL serves the Task YAML"
+
+
+def free_port() -> int:
+    with socket.socket() as probe:
+        probe.bind(("127.0.0.1", 0))
+        return int(probe.getsockname()[1])
+
+
+def alive(pid: int) -> bool:
+    """True while ``pid`` exists and is not a zombie (a killed orphan may wait for its reaper)."""
+    try:
+        return Path(f"/proc/{pid}/stat").read_text().split(")")[-1].split()[0] != "Z"
+    except (FileNotFoundError, ProcessLookupError, IndexError):
+        return False
+
+
+def wait_for(predicate: object, timeout: float = 30.0) -> None:
+    deadline = time.monotonic() + timeout
+    while not predicate():  # type: ignore[operator]
+        assert time.monotonic() < deadline, "timed out"
+        time.sleep(0.05)
+
+
+def spawn_runner(tmp_path: Path, port: int, **extra: str) -> subprocess.Popen[bytes]:
+    """``python -m openultrasast.plane.runner`` as the image runs it: HTTP on, no exit-after-command escape."""
+    child_env = {k: v for k, v in os.environ.items() if k != "AX_RUNNER_EXIT_AFTER_COMMAND"}
+    child_env.update(AX_TASK_YAML=task_yaml(tmp_path / "out", tmp_path / "ws"), AX_WORKSPACES_YAML=files_workspace_yaml())
+    child_env.update(AX_RUNNER_HTTP="1", AX_RUNNER_PORT=str(port), **extra)
+    log = (tmp_path / "runner.log").open("wb")
+    return subprocess.Popen([sys.executable, "-m", "openultrasast.plane.runner"], env=child_env, stdout=log, stderr=log)
+
+
+def test_the_runner_keeps_serving_after_the_command_and_exits_0_on_sigterm(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
+    port = free_port()
+    proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url)
+    marker = tmp_path / "state" / "stub.done.json"
+    try:
+        wait_for(marker.exists)
+        time.sleep(0.5)
+        assert proc.poll() is None, "PID 1 must outlive the command (ax: 'Container guest is stopped' otherwise)"
+        base = f"http://127.0.0.1:{port}"
+        assert get(f"{base}/healthz") == 200 and get(f"{base}/readyz?check=workspace") == 200
+        assert get(f"{base}/metadata/v1alpha1/ax/workspaces") == 200
+        assert len(receiver.posts) == 1
+        assert json.loads(marker.read_text()) | {"summary": None} == {"exit": 0, "status": "done", "delivered": True, "summary": None}
+        proc.send_signal(signal.SIGTERM)
+        assert proc.wait(timeout=10) == 0
+    finally:
+        proc.kill()
+        proc.wait()
+
+
+def test_sigterm_goes_to_the_commands_group_then_sigkill_after_the_grace(env: dict[str, str], receiver: Receiver, tmp_path: Path) -> None:
+    port = free_port()
+    proc = spawn_runner(tmp_path, port, OUSAST_ARTIFACT_URL=receiver.url, STUB_MODE="sleep", OUSAST_TERM_GRACE="1")
+    pids_file = tmp_path / "out" / "pids.json"
+    try:
+        wait_for(pids_file.exists)
+        pids = json.loads(pids_file.read_text())
+        assert all(alive(pid) for pid in pids)
+        started = time.monotonic()
+        proc.send_signal(signal.SIGTERM)
+        assert proc.wait(timeout=15) == 0
+        assert time.monotonic() - started >= 0.9, "the command ignored SIGTERM, so the runner waited out the grace"
+        wait_for(lambda: not any(alive(pid) for pid in pids), timeout=5)
+    finally:
+        proc.kill()
+        proc.wait()
+    assert not (tmp_path / "state" / "stub.done.json").exists(), "an interrupted command is not complete"
+    assert receiver.posts == [], "nothing is delivered for an interrupted command"
+
+
+def test_the_completion_marker_prevents_a_rerun_on_resume(
+    env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    counter = tmp_path / "runs.txt"
+    code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
+    assert code == 0 and len(counter.read_text().splitlines()) == 1 and len(receiver.posts) == 1
+    (tmp_path / "ws" / "candidates.json").write_text("changed by the task\n")
+    code, _ = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
+    assert code == 0
+    assert len(counter.read_text().splitlines()) == 1, "a resumed actor must not repeat billed work"
+    assert len(receiver.posts) == 1, "a delivered result is not delivered again"
+    assert (tmp_path / "ws" / "candidates.json").read_text() == "changed by the task\n", "workspaces are prepared once"
+
+
+def test_a_failed_delivery_is_retried_alone_on_the_next_boot(
+    env: dict[str, str], receiver: Receiver, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    counter = tmp_path / "runs.txt"
+    broken = Receiver(500)
+    try:
+        code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=broken.url, STUB_COUNTER=str(counter))
+    finally:
+        broken.stop()
+    assert code == 2
+    marker = json.loads((tmp_path / "state" / "stub.done.json").read_text())
+    assert marker["delivered"] is False and marker["status"] == "failed" and marker["exit"] == 0
+    code, output = run_main(monkeypatch, tmp_path, files_workspace_yaml(), OUSAST_ARTIFACT_URL=receiver.url, STUB_COUNTER=str(counter))
+    assert code == 0
+    assert len(counter.read_text().splitlines()) == 1, "only the delivery is retried"
+    with tarfile.open(fileobj=io.BytesIO(receiver.posts[0][1]), mode="r:") as archive:
+        summary = archive.extractfile("summary.json")
+        assert summary is not None and json.loads(summary.read())["status"] == "done", "the task's own summary"
+    marker = json.loads((tmp_path / "state" / "stub.done.json").read_text())
+    assert marker["delivered"] is True and marker["status"] == "done"
+
+
 # --- git workspaces from the case cache and by shallow clone ----------------------------------------------------------------