mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
chore(p0): drop never-enabled hunter tools, test-only functions and receiver delivery
Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts). Deleted: the four hunter tools no caller enables (read_definition, entry_points, flows, obligations) with their schemas and dispatch, the dead mapping branch of find_refs; 20 functions whose only caller was one test (mapping task records, compute_ranking_metrics, build_quick_hunter_prompt, validate_evidence_transition, should_fuse, harness processors, rank_targets_with_model, model/report slice rendering, taxonomy hierarchical/related, config_value.verdict, taint.verdict, roles.load_roles, features.language_of and build_for_scan, folds.leave_one_*, features.recorded_case, search BatchExecutor and the three __main__ shims, delivery.put_input); the receiver delivery mode of the plane. push_inputs.py moved to benchmarks/push. memory.purge stays: the live-S3 test cleanup uses it. Net: 2,730 lines deleted, 207 added. 473 targeted tests passed; 3,282 tests collect. breadth.toml records the removals. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
f5a904b7f2
commit
f63703a604
@@ -13,7 +13,6 @@ Store layout under the 30-day ``runs/`` expiry rule (``memory.RUNS_RULE_ID``)::
|
||||
runs/<run>/<task>/output.tar the runner's PUT
|
||||
runs/<run>/<task>/<artifact> each declared output, re-put by the reconciler
|
||||
runs/<run>/<task>/done.json status, bytes, file count, finished
|
||||
runs/<run>/inputs/<name> an input no task produces (``put_input``)
|
||||
|
||||
Nothing here logs or prints a URL.
|
||||
"""
|
||||
@@ -164,11 +163,6 @@ class Delivery:
|
||||
put.append(artifact)
|
||||
return put
|
||||
|
||||
def put_input(self, name: str, path: Path) -> str:
|
||||
"""Store a file no task produces as ``runs/<run>/inputs/<name>``; returns its presigned GET URL."""
|
||||
self.store._put(self.key(INPUTS_DIR, name), Path(path).read_bytes())
|
||||
return self.store.presign_get(self.key(INPUTS_DIR, name), self.expires)
|
||||
|
||||
# --- state ---------------------------------------------------------------------------------------------------
|
||||
|
||||
def save_state(self, data: Mapping[str, Any]) -> None:
|
||||
|
||||
@@ -1,61 +1,29 @@
|
||||
"""Per-task network egress on Agent Substrate (ai-service-plane, task 0: gateway + receiver Service).
|
||||
"""Per-task TLS egress to the store, Git hosts and model endpoints.
|
||||
|
||||
Every TCP connection an actor opens (DNS aside) is redirected to Substrate's egress gateway
|
||||
(``atenet-egress.ate-system``), which denies by default: an actor without an ``EgressPolicy`` gets no tunnel, and
|
||||
a refused connection is accepted and then closed. :func:`policy_for` grants a task exactly what it needs:
|
||||
|
||||
* ``http`` on port 80 to the artifact receiver's cluster name (``ousast-receiver.ax-system.svc.cluster.local``, a
|
||||
selector-less Service whose endpoint is this host's receiver, ``ops/ax/receiver-service.yaml.tmpl``). The actor
|
||||
cannot resolve cluster names, so the runner dials ``OUSAST_ARTIFACT_DIAL`` (the Service's ClusterIP, port 80)
|
||||
with the receiver's name as ``Host``. The gateway decides the policy on that ``Host`` but connects to the
|
||||
address the actor dialled (measured live: agentgateway's upstream is the CONNECT authority, not the Host), so
|
||||
the dial address must be one the gateway pod reaches: the ClusterIP, which kube-proxy maps to the host.
|
||||
* ``tls_passthrough`` on 443 for the Git hosts of the bound Workspaces' ``https`` repos (smart HTTP talks to the
|
||||
repo's host only) and for the bound Model's ``openultrasast.io/egress-hosts`` (``api.deepseek.com``).
|
||||
|
||||
Nothing else: no wildcard, no IP address (the gateway rejects them), no ``https`` rule (that one intercepts TLS and
|
||||
the actor would have to trust the gateway CA, which ax templates do not mount). :class:`Egress` writes the policy
|
||||
with ``kubectl ate create egress-policy`` after ``ax apply`` created the actor, reads it back, and before resume
|
||||
waits ``OUSAST_EGRESS_SETTLE`` s (11) because the gateway caches a missing policy as deny for 10 s. The kubectl-ate
|
||||
CLI has no delete verb: Substrate's store deletes the policy with its actor (``ON DELETE CASCADE``), so
|
||||
:meth:`Egress.delete` runs after ``ax delete task`` and confirms the policy is gone.
|
||||
The gateway denies by default. Egress writes and checks each actor's policy before resume,
|
||||
then waits for the cached denial to expire. Deleting the actor removes its policy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterable, Mapping
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlsplit
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .manifests import Model, Task, Workspace
|
||||
|
||||
__all__ = [
|
||||
"INPUTS_PATH",
|
||||
"RECEIVER_HOST",
|
||||
"RECEIVER_PORT",
|
||||
"Egress",
|
||||
"Receiver",
|
||||
"git_hosts",
|
||||
"receiver_cluster_ip",
|
||||
"policy_for",
|
||||
"receiver_address",
|
||||
"rules_of",
|
||||
]
|
||||
|
||||
RECEIVER_HOST = "ousast-receiver.ax-system.svc.cluster.local"
|
||||
INPUTS_PATH = "/inputs/" # GET <receiver>/inputs/<producer>/<artifact>: a consumer fetches a declared input
|
||||
RECEIVER_PORT = 18090 # the receiver Service's targetPort; ops/ax/up.sh renders the Service with the same number
|
||||
_NOT_FOUND = ("not found", "notfound")
|
||||
|
||||
|
||||
@@ -89,12 +57,10 @@ def policy_for(
|
||||
workspaces: Iterable[Workspace],
|
||||
model: Model | None,
|
||||
store_host: str | None,
|
||||
receiver_host: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""The task's EgressPolicy body (plane-on-kubernetes design section 3): one ``tlsPassthrough`` rule on 443 over
|
||||
the sorted union of the store's hostname (``store_host``: where the runner PUTs its tar and GETs its inputs), the
|
||||
bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. No ``http`` rule, unless
|
||||
``receiver_host`` names the old receiver (the path task 2.6 removes). An http Git repo cannot be reached under
|
||||
bound Workspaces' https Git hosts and the bound Model's ``openultrasast.io/egress-hosts``. An http Git repo cannot be reached under
|
||||
this policy and is refused; a store given as an address is refused naming ``S3_ENDPOINT`` (the gateway allows
|
||||
hostnames only)."""
|
||||
tls, plain = git_hosts(workspaces)
|
||||
@@ -106,8 +72,6 @@ def policy_for(
|
||||
raise ValueError(f"Task/{task.metadata.name}: the store endpoint is an address ({store_host}); S3_ENDPOINT must be a hostname")
|
||||
hosts.add(store_host.lower())
|
||||
rules: list[dict[str, Any]] = []
|
||||
if receiver_host and not _is_ip(receiver_host): # the receiver path, kept for group 2 only (task 2.6 deletes it)
|
||||
rules.append({"http": {"hostnames": [receiver_host.lower()], "ports": {"numbers": [80]}}})
|
||||
if hosts:
|
||||
rules.append({"tlsPassthrough": {"hostnames": sorted(hosts), "ports": {"numbers": [443]}}})
|
||||
return {"rules": rules}
|
||||
@@ -125,41 +89,6 @@ def rules_of(policy: Mapping[str, Any]) -> list[dict[str, Any]]:
|
||||
return sorted(out, key=lambda r: json.dumps(r, sort_keys=True))
|
||||
|
||||
|
||||
def receiver_cluster_ip(context: str) -> str | None:
|
||||
"""The ClusterIP of the ``ousast-receiver`` Service (``kubectl get svc`` in the profile's ``context``); None when
|
||||
kubectl cannot say."""
|
||||
if not context:
|
||||
raise RuntimeError("receiver_cluster_ip needs the profile's kube context")
|
||||
argv = ["kubectl", "--context", context, "-n", "ax-system", "get", "svc", "ousast-receiver", "-o", "jsonpath={.spec.clusterIP}"]
|
||||
try:
|
||||
proc = subprocess.run(argv, capture_output=True, text=True, check=False, timeout=20)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
address = proc.stdout.strip()
|
||||
return address if proc.returncode == 0 and _is_ip(address) else None
|
||||
|
||||
|
||||
def receiver_address(port: int, context: str | None) -> tuple[str, str | None]:
|
||||
"""(``OUSAST_ARTIFACT_URL``, ``OUSAST_ARTIFACT_DIAL``) for a receiver bound on ``port``; ``context`` is the
|
||||
profile's kube context the Service's ClusterIP is looked up in.
|
||||
|
||||
``OUSAST_ARTIFACT_HOST`` set: the old direct URL ``http://<host>:<port>/`` and no dial (tests, a host the actor
|
||||
reaches without the gateway). Otherwise the receiver Service's name, dialled at ``OUSAST_ARTIFACT_DIAL`` or the
|
||||
Service's ClusterIP on port 80: the gateway checks the policy against ``Host`` but connects to the address the
|
||||
actor dialled (its CONNECT authority), so that address must reach the receiver from the gateway pod.
|
||||
"""
|
||||
host = os.environ.get("OUSAST_ARTIFACT_HOST")
|
||||
if host:
|
||||
return f"http://{host}:{port}/", None
|
||||
dial = os.environ.get("OUSAST_ARTIFACT_DIAL")
|
||||
if not dial:
|
||||
if not context:
|
||||
raise RuntimeError("receiver_address needs the profile's kube context to look the receiver Service up")
|
||||
address = receiver_cluster_ip(context)
|
||||
dial = f"{address}:80" if address else None
|
||||
return f"http://{RECEIVER_HOST}/", dial
|
||||
|
||||
|
||||
class Egress:
|
||||
"""``kubectl ate`` for one actor's egress policy; the executable is injectable so tests use a fake."""
|
||||
|
||||
@@ -230,95 +159,3 @@ class Egress:
|
||||
time.sleep(float(os.environ.get("OUSAST_EGRESS_SETTLE") or 11))
|
||||
|
||||
|
||||
class Receiver(ThreadingHTTPServer):
|
||||
"""Accepts one tar per running task and extracts it under ``<run dir>/<task>/`` (POST); serves a running task
|
||||
the producer artifacts it declared as inputs (``GET /inputs/<producer>/<artifact>``, ``inputs[task]``)."""
|
||||
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, run_name: str, base: Path, port: int, context: str | None = None) -> None:
|
||||
super().__init__(("0.0.0.0", port), _Handler)
|
||||
self.run_name, self.base, self.running, self.delivered = run_name, base, set[str](), set[str]()
|
||||
self.inputs: dict[str, frozenset[str]] = {}
|
||||
self.lock = threading.Lock()
|
||||
self.url, self.dial = receiver_address(self.server_address[1], context)
|
||||
|
||||
def _task(self, headers: Any) -> str | None:
|
||||
"""The ``X-Ousast-Task`` of a request naming this run and a task that is running now, else None."""
|
||||
task = str(headers.get("X-Ousast-Task", ""))
|
||||
with self.lock:
|
||||
return task if headers.get("X-Ousast-Run") == self.run_name and task in self.running else None
|
||||
|
||||
def input_file(self, headers: Any, ref: str) -> tuple[int, str | Path]:
|
||||
"""(200, the file) when ``ref`` is one of the requesting task's declared inputs, else (403/404, why).
|
||||
|
||||
``ref`` must equal a declared ``<producer>/<artifact>`` exactly and resolve to a regular file under the run
|
||||
directory; nothing is normalised first, so ``..`` or an absolute path never names a declared input."""
|
||||
task = self._task(headers)
|
||||
if task is None:
|
||||
return 403, "unknown run or task"
|
||||
with self.lock:
|
||||
declared = ref in self.inputs.get(task, frozenset())
|
||||
if not declared:
|
||||
return 403, f"{ref!r} is not an input of {task}"
|
||||
root = self.base.resolve()
|
||||
path = (root / ref).resolve()
|
||||
if not path.is_relative_to(root) or not path.is_file():
|
||||
return 404, f"{ref} was not delivered"
|
||||
return 200, path
|
||||
|
||||
def accept(self, headers: Any, body: bytes) -> tuple[int, str]:
|
||||
task = self._task(headers)
|
||||
if task is None:
|
||||
return 403, "unknown run or task"
|
||||
if not str(headers.get("Content-Type", "")).startswith("application/x-tar"):
|
||||
return 415, "expected application/x-tar"
|
||||
dest = (self.base / task).resolve()
|
||||
try:
|
||||
with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as tar:
|
||||
members = tar.getmembers()
|
||||
for m in members:
|
||||
if m.issym() or m.islnk() or not (dest / m.name).resolve().is_relative_to(dest):
|
||||
return 400, f"rejected member {m.name}"
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
tar.extractall(dest, members) # every member checked above
|
||||
except tarfile.TarError as exc:
|
||||
return 400, f"bad tar: {exc}"
|
||||
with self.lock:
|
||||
self.delivered.add(task)
|
||||
return 200, "ok"
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None: # noqa: N802 - http.server's naming
|
||||
server: Receiver = self.server # type: ignore[assignment]
|
||||
path = urlsplit(self.path).path
|
||||
if not path.startswith(INPUTS_PATH):
|
||||
self._text(404, "not found")
|
||||
return
|
||||
code, found = server.input_file(self.headers, unquote(path[len(INPUTS_PATH) :]))
|
||||
if not isinstance(found, Path):
|
||||
self._text(code, found)
|
||||
return
|
||||
with found.open("rb") as source: # streamed: a producer artifact can be far larger than any env value
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(os.fstat(source.fileno()).st_size))
|
||||
self.end_headers()
|
||||
shutil.copyfileobj(source, self.wfile, 1 << 16)
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802 - http.server's naming
|
||||
server: Receiver = self.server # type: ignore[assignment]
|
||||
body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
|
||||
self._text(*server.accept(self.headers, body))
|
||||
|
||||
def _text(self, code: int, text: str) -> None:
|
||||
data = text.encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "text/plain")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
@@ -10,8 +10,7 @@ credential from this process's environment, never from a file (Req 4.6), and the
|
||||
``delivery.py`` (plane-on-kubernetes design section 3). Completion is the runner's output tar appearing in the
|
||||
memory store (ax has no Completed phase), which the reconciler polls, downloads, extracts and re-publishes per
|
||||
declared output; ``Failed``, the task vanishing, a refused start or ``OUSAST_TASK_TIMEOUT`` s (7200) without
|
||||
delivery fail it; ``ax delete task`` follows. ``OUSAST_DELIVERY=receiver`` keeps the host receiver of the kind
|
||||
profile for group 2 only (task 2.6 removes it).
|
||||
delivery fail it; ``ax delete task`` follows.
|
||||
|
||||
ax rejects unknown fields, so rendered documents carry only ax's fields; ``openultrasast.io/model`` reaches the
|
||||
task as ``OUSAST_MODEL`` plus ``OUSAST_MODEL_PARAMS`` (Req 7.1), each Workspace's pin annotation as
|
||||
@@ -40,13 +39,12 @@ from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yaml
|
||||
|
||||
from .delivery import Delivery, DeliveryError, declared
|
||||
from .doctor import doctor
|
||||
from .egress import RECEIVER_PORT, Egress, Receiver, policy_for
|
||||
from .egress import Egress, policy_for
|
||||
from .manifests import AX_API_VERSION, Manifests, Model, Run, RunTask, Task, Workspace, load_manifests
|
||||
from .memory import open_store
|
||||
from .profile import PlaneProfile, load_profile
|
||||
@@ -142,16 +140,14 @@ def _model_doc(model: Model) -> dict[str, Any]:
|
||||
return _doc("Model", model.metadata.name, spec, model.metadata.atespace)
|
||||
|
||||
|
||||
def render_task(run: Run, entry: RunTask, manifests: Manifests, artifact_url: str = "", dial: str | None = None) -> list[dict[str, Any]]:
|
||||
def render_task(run: Run, entry: RunTask, manifests: Manifests) -> list[dict[str, Any]]:
|
||||
"""The documents ``ax apply`` receives for one Run task: its Model, its Workspaces, the Task last. Task-produced
|
||||
inputs are never rendered (Substrate caps one env value at 32768 characters): the runner fetches them
|
||||
(``OUSAST_INPUTS``) after the start by presigned URL, or from the receiver when ``artifact_url`` is given."""
|
||||
(``OUSAST_INPUTS``) after the start by presigned URL."""
|
||||
task: Task = manifests.tasks[entry.task]
|
||||
ax_name = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT)
|
||||
env = {e.name: e.value for e in task.env}
|
||||
env.update(OUSAST_RUN=run.metadata.name, OUSAST_TASK=entry.name, OUSAST_OUTPUT_DIR=f"{OUTPUT_ROOT}/{entry.name}")
|
||||
if artifact_url: # receiver mode only (task 2.6 removes it): the store path carries no URL in the env
|
||||
env.update(OUSAST_ARTIFACT_URL=artifact_url, **({"OUSAST_ARTIFACT_DIAL": dial} if dial else {}))
|
||||
budget = _fields(entry.budget) if entry.budget else {}
|
||||
env.update({f"OUSAST_BUDGET_{k.upper()}": str(v) for k, v in budget.items()})
|
||||
inputs = {re.sub(r"[^A-Z0-9]+", "_", name.upper()): ref for name, ref in entry.inputs.items()}
|
||||
@@ -341,25 +337,22 @@ def _execute(
|
||||
base: Path,
|
||||
delivery: Delivery,
|
||||
router: Router,
|
||||
receiver: Receiver | None = None,
|
||||
) -> tuple[str, str]:
|
||||
ax_name, task = _ax_name(run.metadata.name, entry.name, limit=TASK_NAME_LIMIT), manifests.tasks[entry.task]
|
||||
atespace, manifest = task.metadata.atespace or "default", base / entry.name / "task.yaml"
|
||||
manifest.parent.mkdir(parents=True, exist_ok=True)
|
||||
docs = render_task(run, entry, manifests, receiver.url if receiver else "", receiver.dial if receiver else None)
|
||||
docs = render_task(run, entry, manifests)
|
||||
manifest.write_text(yaml.safe_dump_all(docs, sort_keys=False), encoding="utf-8")
|
||||
model = manifests.models.get(task.metadata.annotations.get(MODEL_ANNOTATION) or "")
|
||||
workspaces = [manifests.workspaces[b.name] for b in task.workspaces]
|
||||
body = None if receiver else delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body
|
||||
store_host, receiver_host = (None, urlsplit(receiver.url).hostname) if receiver else (delivery.host, None)
|
||||
if receiver is None:
|
||||
delivery.reset(entry.name) # a previous attempt's tar must not complete this one
|
||||
body = delivery.body(entry.name, entry.inputs) # presigned URLs ride in the start body
|
||||
delivery.reset(entry.name) # a previous attempt's tar must not complete this one
|
||||
|
||||
def delivered() -> bool:
|
||||
return entry.name in receiver.delivered if receiver is not None else delivery.delivered(entry.name)
|
||||
return delivery.delivered(entry.name)
|
||||
|
||||
try: # Req 4.6: the bound Model's credential is read here and only ever sent in the start request
|
||||
policy = policy_for(task, workspaces, model, store_host, receiver_host)
|
||||
policy = policy_for(task, workspaces, model, delivery.host)
|
||||
start = router.starter(atespace, ax_name, run.metadata.name, credentials(model), body)
|
||||
ax.apply(manifest) # creates the Substrate actor; its egress is denied until the policy exists
|
||||
egress.apply(ax_name, atespace, policy)
|
||||
@@ -369,7 +362,7 @@ def _execute(
|
||||
return "failed", str(exc)
|
||||
try:
|
||||
failure = _await(ax, ax_name, delivered, start)
|
||||
if failure is None and receiver is None:
|
||||
if failure is None:
|
||||
delivery.collect(entry.name, declared(run)[entry.name]) # download, extract, re-publish what is declared
|
||||
except DeliveryError as exc:
|
||||
failure = str(exc)
|
||||
@@ -408,11 +401,6 @@ def run(
|
||||
_read_json(base / "state.json") or {"run": run_spec.metadata.name, "started": _now(), "tasks": {}},
|
||||
mirror=delivery.save_state,
|
||||
)
|
||||
receiver: Receiver | None = None
|
||||
if os.environ.get("OUSAST_DELIVERY") == "receiver": # the kind profile's host receiver, group 2 only (2.6 removes it)
|
||||
port = os.environ.get("OUSAST_ARTIFACT_PORT") or (0 if os.environ.get("OUSAST_ARTIFACT_HOST") else RECEIVER_PORT)
|
||||
receiver = Receiver(run_spec.metadata.name, base, int(port), profile.kube_context) # the Service targets RECEIVER_PORT
|
||||
threading.Thread(target=receiver.serve_forever, daemon=True).start()
|
||||
cli, egress, finished = Ax(ax), Egress(kubectl_ate, profile.kube_context), queue.Queue[tuple[str, str, str]]()
|
||||
pending = [t for t in run_spec.tasks if state.status(t.name) != "done"]
|
||||
for entry in run_spec.tasks:
|
||||
@@ -427,7 +415,7 @@ def run(
|
||||
cli.delete(_ax_name(run_spec.metadata.name, entry.name, limit=TASK_NAME_LIMIT)) # an interrupted attempt; ax must not keep it
|
||||
state.set(entry.name, status="running", started=_now(), finished=None)
|
||||
try:
|
||||
outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router, receiver)
|
||||
outcome = _execute(cli, egress, run_spec, entry, manifests, base, delivery, router)
|
||||
except Exception as exc: # noqa: BLE001 - a worker that dies silently would hang the run on the queue
|
||||
outcome = ("failed", f"{type(exc).__name__}: {exc}")
|
||||
finished.put((entry.name, *outcome))
|
||||
@@ -444,27 +432,17 @@ def run(
|
||||
pending.remove(entry)
|
||||
running[entry.name] = entry
|
||||
blocked.add(entry.serialize or "")
|
||||
if receiver is not None:
|
||||
with receiver.lock: # the receiver serves a running task its declared inputs, nothing else
|
||||
receiver.running.add(entry.name)
|
||||
receiver.inputs[entry.name] = frozenset(entry.inputs.values())
|
||||
threading.Thread(target=worker, args=(entry,), daemon=True).start()
|
||||
if not running:
|
||||
break # nothing can start: a producer stopped short
|
||||
name, status, reason = finished.get()
|
||||
del running[name]
|
||||
if receiver is not None:
|
||||
with receiver.lock:
|
||||
receiver.running.discard(name)
|
||||
receiver.inputs.pop(name, None)
|
||||
summary = _read_json(base / name / "summary.json") or {}
|
||||
state.set(name, status=status, finished=_now(), usd=summary.get("usd"), model=summary.get("model"), reason=reason or None)
|
||||
halted = halted or status == "failed"
|
||||
if status == "unfinished":
|
||||
pending = [t for t in pending if name not in t.producers]
|
||||
finally:
|
||||
if receiver is not None:
|
||||
receiver.shutdown()
|
||||
lock.unlink(missing_ok=True)
|
||||
statuses = [state.status(t.name) for t in run_spec.tasks]
|
||||
result = "failed" if "failed" in statuses else "done" if all(s == "done" for s in statuses) else "unfinished"
|
||||
@@ -546,4 +524,4 @@ def status(run_name: str, *, units: bool = False, results_root: Path | None = No
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
__all__ = ["Ax", "Receiver", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"]
|
||||
__all__ = ["Ax", "attribution", "doctor", "load_run", "mark_done", "outcome_of", "render_task", "run", "run_dir", "status"]
|
||||
|
||||
@@ -25,8 +25,7 @@ The command runs as a child process, ``python -m openultrasast.plane.tasks.<spec
|
||||
in its own process group, with the first workspace as its working directory and ``AX_METADATA_URL`` pointing at
|
||||
this server. ax has no artifact channel and never reports that a command finished, so after the child exits the
|
||||
runner reads ``summary.json`` (a crash without one becomes ``failed`` with the exit code and the stderr tail),
|
||||
PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` (or POSTs it to ``OUSAST_ARTIFACT_URL``, the receiver
|
||||
path kept until task 2.6 removes it) -- that delivery is the task's
|
||||
PUTs the whole ``OUSAST_OUTPUT_DIR`` as a tar to ``delivery.put`` -- that delivery is the task's
|
||||
completion (Requirement 4.4); a failed one is reported ``failed``, never done -- and writes the completion marker
|
||||
``<OUSAST_STATE_DIR>/<task>.done.json`` (``exit``, ``status``, ``delivered``). Then it keeps serving until SIGTERM,
|
||||
as ax requires of PID 1. A boot that finds the marker never reruns the command (a resumed actor must not repeat
|
||||
@@ -48,13 +47,9 @@ Environment the runner reads, all optional except the two ax variables:
|
||||
OUSAST_TERM_GRACE seconds between SIGTERM and SIGKILL of the command's group, default 10
|
||||
OUSAST_CASE_CACHE a directory of git checkouts (``benchmarks/independent`` cache layout)
|
||||
OUSAST_GIT_PINS JSON ``{"<workspace>/<git name>": "<40-hex commit>"}`` from the reconciler
|
||||
OUSAST_ARTIFACT_URL, OUSAST_RUN where to deliver, and the run name carried in ``X-Ousast-Run``
|
||||
OUSAST_ARTIFACT_DIAL ``host:port`` to connect to instead of the URL's host, which is sent as
|
||||
``Host`` (the egress gateway decides on and resolves that name)
|
||||
OUSAST_DELIVERY_BACKOFF seconds before the second delivery attempt, doubled per retry
|
||||
OUSAST_INPUTS JSON ``{"<NAME>": "<producer>/<artifact>"}``: after the start and before the
|
||||
command each is fetched from ``delivery.inputs[NAME]`` (else from
|
||||
``<OUSAST_ARTIFACT_URL>/inputs/<ref>`` with the receiver's dial and headers)
|
||||
command each is fetched from ``delivery.inputs[NAME]``
|
||||
to ``OUSAST_INPUT_<NAME>``, with the same retries as delivery; a failed
|
||||
fetch delivers a failed summary naming the input, the command never runs
|
||||
|
||||
@@ -700,16 +695,6 @@ def build_tar(output_dir: Path) -> bytes:
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _dialled(url: str, headers: Mapping[str, str], dial: str | None) -> tuple[str, dict[str, str]]:
|
||||
"""(URL to connect to, headers): with ``dial`` the connection goes there and the URL's host is sent as ``Host``."""
|
||||
out = dict(headers)
|
||||
if not dial:
|
||||
return url, out
|
||||
parts = urlsplit(url)
|
||||
out["Host"] = parts.netloc
|
||||
return parts._replace(netloc=dial).geturl(), out
|
||||
|
||||
|
||||
def safe_url(url: str) -> str:
|
||||
"""``scheme://host/path`` of a URL: what a log line or an error may carry (a presigned query is a credential)."""
|
||||
parts = urlsplit(url)
|
||||
@@ -723,18 +708,14 @@ def deliver(
|
||||
*,
|
||||
attempts: int = DELIVERY_ATTEMPTS,
|
||||
backoff: float = 1.0,
|
||||
dial: str | None = None,
|
||||
method: str = "POST",
|
||||
) -> int:
|
||||
"""Send ``payload`` to ``url`` (``POST`` to the receiver, ``PUT`` to a presigned store URL) with ``headers``; retry
|
||||
with doubling backoff; return the 2xx status. Log lines and errors carry the URL without its query.
|
||||
"""PUT the payload with retries and doubling backoff; return the 2xx status.
|
||||
|
||||
``dial`` (``host:port``): connect there instead and send the URL's host as ``Host``, as the actor must -- it
|
||||
cannot resolve the receiver's cluster name, and the egress gateway decides on ``Host`` and resolves it itself.
|
||||
Log lines and errors carry the URL without its query.
|
||||
"""
|
||||
last, (target, headers) = "no attempt made", _dialled(url, headers, dial)
|
||||
last = "no attempt made"
|
||||
for attempt in range(1, attempts + 1):
|
||||
request = urllib.request.Request(target, data=payload, method=method, headers=headers)
|
||||
request = urllib.request.Request(url, data=payload, method="PUT", headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - the reconciler's URL
|
||||
status = int(response.status)
|
||||
@@ -752,16 +733,16 @@ def deliver(
|
||||
|
||||
|
||||
def fetch(
|
||||
url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0, dial: str | None = None
|
||||
url: str, headers: Mapping[str, str], dest: Path, *, attempts: int = DELIVERY_ATTEMPTS, backoff: float = 1.0
|
||||
) -> int:
|
||||
"""GET ``url`` into ``dest`` (streamed to a sibling temp file, then renamed); retried like :func:`deliver`.
|
||||
|
||||
Returns the byte count; :class:`DeliveryError` names the last cause when every attempt failed."""
|
||||
last, (target, headers) = "no attempt made", _dialled(url, headers, dial)
|
||||
last = "no attempt made"
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
partial = dest.with_name(dest.name + ".part")
|
||||
for attempt in range(1, attempts + 1):
|
||||
request = urllib.request.Request(target, method="GET", headers=headers)
|
||||
request = urllib.request.Request(url, method="GET", headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as out: # noqa: S310
|
||||
shutil.copyfileobj(response, out, 1 << 16)
|
||||
@@ -784,7 +765,7 @@ def fetch(
|
||||
|
||||
def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> list[Path]:
|
||||
"""Fetch every ``OUSAST_INPUTS`` entry to its ``OUSAST_INPUT_<NAME>`` path: from ``delivery.inputs[NAME]`` (a
|
||||
presigned GET, no headers) when the start request carries one, else from the receiver.
|
||||
presigned GET, no headers) carried by the start request.
|
||||
|
||||
:class:`RunnerError` names the input when one cannot be fetched: a task never runs with an input missing."""
|
||||
raw = environ.get("OUSAST_INPUTS")
|
||||
@@ -796,21 +777,15 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None =
|
||||
raise RunnerError(f"OUSAST_INPUTS is not JSON: {exc}") from None
|
||||
if not isinstance(inputs, dict):
|
||||
raise RunnerError("OUSAST_INPUTS must map input names to <producer>/<artifact>")
|
||||
url = environ.get("OUSAST_ARTIFACT_URL")
|
||||
headers = {"X-Ousast-Run": environ.get("OUSAST_RUN", ""), "X-Ousast-Task": environ.get("OUSAST_TASK", "")}
|
||||
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
||||
fetched: list[Path] = []
|
||||
for name, ref in sorted(inputs.items()):
|
||||
dest = environ.get(f"OUSAST_INPUT_{name}")
|
||||
presigned = delivery.inputs.get(name) if delivery is not None else None
|
||||
if not dest or not isinstance(ref, str) or not (presigned or url):
|
||||
raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL, OUSAST_ARTIFACT_URL or OUSAST_INPUT_{name}")
|
||||
if not dest or not isinstance(ref, str) or not presigned:
|
||||
raise RunnerError(f"input {name} ({ref}) not fetched: no delivery URL or OUSAST_INPUT_{name}")
|
||||
try:
|
||||
if presigned:
|
||||
size = fetch(presigned, {}, Path(dest), backoff=backoff)
|
||||
else:
|
||||
source = f"{str(url).rstrip('/')}/inputs/{urllib.parse.quote(ref)}"
|
||||
size = fetch(source, headers, Path(dest), backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
|
||||
size = fetch(presigned, {}, Path(dest), backoff=backoff)
|
||||
except DeliveryError as exc:
|
||||
raise RunnerError(f"input {name} ({ref}) not fetched: {exc}") from None
|
||||
log.info("input %s (%s): %d bytes to %s", name, ref, size, dest)
|
||||
@@ -818,23 +793,14 @@ def fetch_inputs(environ: Mapping[str, str], delivery: DeliveryTargets | None =
|
||||
return fetched
|
||||
|
||||
|
||||
def _deliver_output(task: Task, output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool:
|
||||
"""PUT the output tar to ``delivery.put`` when the start request carried one, else POST it to the receiver;
|
||||
def _deliver_output(output_dir: Path, environ: Mapping[str, str], delivery: DeliveryTargets | None = None) -> bool:
|
||||
"""PUT the output tar to ``delivery.put`` when the start request carried one;
|
||||
False when there is nowhere to deliver; :class:`DeliveryError` when it fails."""
|
||||
backoff = float(environ.get("OUSAST_DELIVERY_BACKOFF", "1.0"))
|
||||
if delivery is not None and delivery.put:
|
||||
deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff, method="PUT")
|
||||
deliver(delivery.put, build_tar(output_dir), {"Content-Type": "application/x-tar"}, backoff=backoff)
|
||||
return True
|
||||
url = environ.get("OUSAST_ARTIFACT_URL")
|
||||
if not url:
|
||||
return False
|
||||
# the Run's task name (``OUSAST_TASK``) is what the receiver knows; the ax Task is named ``<run>-<task>``
|
||||
headers = {"Content-Type": "application/x-tar", "X-Ousast-Task": environ.get("OUSAST_TASK") or task.metadata.name}
|
||||
run = environ.get("OUSAST_RUN")
|
||||
if run:
|
||||
headers["X-Ousast-Run"] = run
|
||||
deliver(url, build_tar(output_dir), headers, backoff=backoff, dial=environ.get("OUSAST_ARTIFACT_DIAL") or None)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# --- entrypoint --------------------------------------------------------------------------------------------------
|
||||
@@ -849,7 +815,7 @@ class _Session:
|
||||
|
||||
@property
|
||||
def marker(self) -> Path:
|
||||
"""``<state>/<task>.done.json``: the command finished; ``delivered`` says whether the receiver has it."""
|
||||
"""``<state>/<task>.done.json``: the command finished; ``delivered`` says whether the store has it."""
|
||||
return self.state_dir / f"{self.task.metadata.name}.done.json"
|
||||
|
||||
@property
|
||||
@@ -893,7 +859,7 @@ def _finish(
|
||||
"""
|
||||
marker: dict[str, Any] = {"exit": exit_code, "status": str(summary.get("status")), "delivered": False, "summary": dict(summary)}
|
||||
try:
|
||||
marker["delivered"] = _deliver_output(session.task, session.output_dir, env, delivery)
|
||||
marker["delivered"] = _deliver_output(session.output_dir, env, delivery)
|
||||
except DeliveryError as exc:
|
||||
log.error("artifact delivery failed: %s", exc)
|
||||
marker.update(status="failed", reason=f"artifact delivery failed: {exc}")
|
||||
@@ -938,7 +904,7 @@ def _run_once(env: MutableMapping[str, str], ready: threading.Event, stop: threa
|
||||
return 0
|
||||
try:
|
||||
_materialise_once(session, env)
|
||||
fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it (and a receiver serves a running task only)
|
||||
fetch_inputs(env, start.delivery) # after the start: the URLs arrive with it
|
||||
except RunnerError as exc:
|
||||
log.error("runner failed: %s", exc)
|
||||
failed = _failed(f"runner failed: {exc}", read_summary(session.output_dir))
|
||||
|
||||
@@ -199,22 +199,6 @@ def _model(summary: object) -> str | None:
|
||||
return str(summary["model"]) if isinstance(summary, dict) and summary.get("model") else None
|
||||
|
||||
|
||||
def recorded_case(base: Path, case: str, workspace: Path | None = None) -> list[dict[str, Any]]:
|
||||
""":func:`case_features` over a recorded run directory's artifacts (the final ``agreed.json`` when present)."""
|
||||
final = base / f"{case}-final" / "agreed.json"
|
||||
return case_features(
|
||||
facts=_json(base / f"{case}-facts" / "facts.json"),
|
||||
passes={p: _jsonl(base / f"{case}-v{p}" / "units.jsonl") for p in PASSES},
|
||||
models={p: _model(_json(base / f"{case}-v{p}" / "summary.json")) for p in PASSES},
|
||||
agreed=_json(final if final.is_file() else base / f"{case}-agree" / "agreed.json"),
|
||||
alerts=_jsonl(base / f"{case}-alerts" / "alerts.jsonl"),
|
||||
alerts_summary=_json(base / f"{case}-alerts" / "summary.json"),
|
||||
workspace=workspace,
|
||||
facts_summary=_json(base / f"{case}-facts" / "summary.json"),
|
||||
model_roles=_json(base / f"{case}-roles" / "roles.json"),
|
||||
)
|
||||
|
||||
|
||||
def _path(env: Mapping[str, str], name: str) -> Path | None:
|
||||
return Path(env[name]) if env.get(name) else None
|
||||
|
||||
@@ -266,7 +250,7 @@ def main(environ: Mapping[str, str] | None = None) -> int:
|
||||
return 0 if summary["status"] == "done" else 2
|
||||
|
||||
|
||||
__all__ = ["DROPPED_ON_READ", "case_features", "main", "recorded_case"]
|
||||
__all__ = ["DROPPED_ON_READ", "case_features", "main", ]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user