mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 20:49:28 +00:00
Marius CiepluchandClaude Opus 5.5
bb5c534fb2
The first live run on the local ax cluster and ax's source (docs/runner.md,
docs/sandbox.md, internal/controller/reconciler.go, internal/server/server.go)
showed a lifecycle the task-3/4 code did not model:
- `ax apply` creates a Task Suspended (Ready=False, TaskSuspended); it runs only
after `ax resume task`. The first resume of a new image can fail with
DeadlineExceeded while Agent Substrate builds the golden snapshot, and ax's
server then stores phase Failed; a later resume succeeds.
- ax has no Completed phase and never reads a command's exit back.
- The runner is PID 1 and must keep serving after the command exits; stop and
suspend deliver SIGTERM, to be forwarded to the command's process group.
Our runner imported the task in-process and exited, which stopped the
container ("Container guest is stopped") and made every resume time out.
- ax's controller probes /readyz?check=workspace; the query string made our
handler answer 404.
Runner (Req 4.1-4.4): spec.command runs as a child
`python -m openultrasast.plane.tasks.<name>` in its own process group, cwd the
first workspace, AX_METADATA_URL set; ax's metadata paths are served. After the
child exits the runner reads summary.json (a crash without one becomes failed
with the exit code and stderr tail), delivers the output directory, writes
/workspace/.ousast-state/<task>.done.json and serves until SIGTERM (forwarded
to the group, SIGKILL after OUSAST_TERM_GRACE, exit 0). A boot with the marker
never reruns the command; a failed delivery is retried alone. Workspaces are
prepared on the first boot only. AX_RUNNER_EXIT_AFTER_COMMAND=1 is the
unit-test escape.
Reconciler (Req 2.2, 2.3, 3.1-3.3): `ax resume task` after apply, retrying
DeadlineExceeded/Unavailable with backoff up to OUSAST_RESUME_TIMEOUT (900 s);
completion is the artifact delivery; Failed after a resume went through, a
vanished task, or no delivery within OUSAST_TASK_TIMEOUT (7200 s) fail the
task with ax's condition message; `ax delete task` follows. Ax gains resume()
and phase() returns the condition message. doctor moves to plane/doctor.py
(tasks.md task 0 allows the sibling) to keep reconciler.py under 500 lines.
Tests: the fake ax follows the real lifecycle (Suspended, a failing first
resume that leaves Failed, Running forever, delivery only after a resume went
through), plus Failed-with-message, vanish, resume-timeout and task-timeout
scenarios; runner tests cover the child's cwd/group/metadata URL, stay-up and
SIGTERM in a real subprocess, SIGKILL of a SIGTERM-ignoring group, and the
marker. The opt-in live test now checks resume, Running, suspend and a second
resume instead of a Completed phase. design.md and tasks.md state the
lifecycle; the module audit manifest counts plane.doctor.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
53 lines
2.3 KiB
Python
53 lines
2.3 KiB
Python
"""``ousast plane doctor``: four checks of this host's ax before a Run is submitted (ai-service-plane Req 3.5).
|
|
|
|
A sibling of ``reconciler.py`` (tasks.md task 0 allows either), which re-exports :func:`doctor`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import subprocess
|
|
|
|
UP = "ops/ax/up.sh brings it up"
|
|
|
|
|
|
# --- checks ----------------------------------------------------------------------------------------------
|
|
|
|
|
|
def _sh(*args: str) -> tuple[bool, str]:
|
|
try:
|
|
proc = subprocess.run(args, capture_output=True, text=True, timeout=60, check=False)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
return False, str(exc)
|
|
return proc.returncode == 0, (proc.stdout + proc.stderr).strip()
|
|
|
|
|
|
def _pods_ready(namespace: str, context: str) -> tuple[bool, str]:
|
|
ok, out = _sh("kubectl", "--context", context, "-n", namespace, "get", "pods", "--no-headers")
|
|
if not ok or not out:
|
|
return False, out or "no pods"
|
|
rows = [r for r in (line.split() for line in out.splitlines()) if len(r) > 2 and re.fullmatch(r"\d+/\d+", r[1])]
|
|
if not rows: # "No resources found" is not a ready namespace
|
|
return False, f"no pods in {namespace}"
|
|
bad = [r[0] for r in rows if r[2] not in ("Completed", "Succeeded") and r[1].split("/")[0] != r[1].split("/")[1]]
|
|
return not bad, f"not ready: {', '.join(bad)}" if bad else f"{len(rows)} pods ready"
|
|
|
|
|
|
def doctor() -> list[tuple[str, bool, str]]:
|
|
"""Four checks of the host's ax: kind, Agent Substrate, the ax controller, the runner image (Req 3.5)."""
|
|
context = "kind-" + (os.environ.get("KIND_CLUSTER_NAME") or "ousast")
|
|
ok, out = _sh("kubectl", "--context", context, "cluster-info")
|
|
checks = [("kind cluster", ok, "reachable" if ok else f"context {context} unreachable ({out[:80]}); {UP}")]
|
|
for label, ns in (("agent substrate (ate-system)", "ate-system"), ("ax controller (ax-system)", "ax-system")):
|
|
ok, out = _pods_ready(ns, context)
|
|
checks.append((label, ok, out if ok else f"{out}; {UP}"))
|
|
ok, out = _sh("curl", "-s", "localhost:5001/v2/_catalog")
|
|
present = ok and "ousast-runner" in out
|
|
text = "ousast-runner present" if present else f"missing from localhost:5001 ({out[:80]}); {UP} and loads it"
|
|
checks.append(("runner image in kind registry", present, text))
|
|
return checks
|
|
|
|
|
|
__all__ = ["doctor"]
|