mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(plane): per-task egress policy; receiver by name through the gateway
Agent Substrate's egress gateway denies an actor without an EgressPolicy, so no task could deliver its artifacts or reach a model. egress.py grants each task exactly: http/80 to ousast-receiver.ax-system.svc.cluster.local, and TLS passthrough/443 to the bound Workspaces' https Git hosts and the bound Model's openultrasast.io/egress-hosts (api.deepseek.com for deepseek-flash). No IPs, no wildcards, no https (MITM) rules. The reconciler writes the policy with kubectl-ate between ax apply and resume, reads it back, waits OUSAST_EGRESS_SETTLE (11 s, the gateway's cached deny), and after ax delete task confirms the policy went with its actor (kubectl-ate has no delete verb; the store cascades). The receiver is a selector-less Service (ops/ax/receiver-service.yaml.tmpl, applied by up.sh together with the no-build agentgateway egress gateway) on port 18090 of the host. Measured live: the gateway checks the policy on Host but connects to the address the actor dialled, so the runner dials the Service's ClusterIP (OUSAST_ARTIFACT_DIAL) with the receiver's name as Host. The runner now labels its delivery with the Run's task name (OUSAST_TASK), which is what the receiver accepts; the ax Task name was refused with 403. The Receiver moved to egress.py to keep reconciler.py at 460 lines. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
0744a157bf
commit
137c6c8ed5
@@ -62,3 +62,51 @@ def _scratch_under_tmp_path(monkeypatch: pytest.MonkeyPatch, tmp_path_factory: p
|
||||
|
||||
scratch = tmp_path_factory.mktemp("scratch")
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(scratch))
|
||||
|
||||
|
||||
FAKE_KUBECTL_ATE = r'''#!{python}
|
||||
"""Fake kubectl-ate for egress policies: an actor exists while ``applied/<actor>.json`` exists and is not gone
|
||||
(the fake ax's record); its policy goes with it, as Substrate's store cascades. Every call is logged to ax.log."""
|
||||
import json, sys, time
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
argv = sys.argv[1:]
|
||||
if argv[:1] == ["--context"]:
|
||||
argv = argv[2:]
|
||||
verb, actor, space = argv[0], argv[2], argv[argv.index("-a") + 1]
|
||||
with (HERE / "ax.log").open("a") as log:
|
||||
log.write(f"{time.monotonic():.4f} egress {verb} {actor}\n")
|
||||
record = HERE / "applied" / f"{actor}.json"
|
||||
alive = record.exists() and not json.loads(record.read_text()).get("gone")
|
||||
store = HERE / "policies" / f"{space}_{actor}.json"
|
||||
if not alive:
|
||||
store.unlink(missing_ok=True)
|
||||
sys.exit(print(f'Error: actor "{actor}" in atespace "{space}" not found', file=sys.stderr) or 1)
|
||||
if verb == "get":
|
||||
if not store.exists(): # kubectl-ate: no policy is a valid state, exit 0 with nothing on stdout
|
||||
sys.exit(print(f'actor "{actor}" in atespace "{space}" has no egress policy', file=sys.stderr) or 0)
|
||||
print(store.read_text())
|
||||
elif verb in ("create", "update"):
|
||||
body = json.loads(sys.stdin.read())
|
||||
old = json.loads(store.read_text()) if store.exists() else None
|
||||
if verb == "create" and old is not None:
|
||||
sys.exit(print("Error: already exists", file=sys.stderr) or 1)
|
||||
version = 1 if old is None else old["metadata"]["version"] + 1
|
||||
body["metadata"] = {"name": "default", "atespace": space, "uid": "uid-1", "version": version}
|
||||
store.parent.mkdir(exist_ok=True)
|
||||
store.write_text(json.dumps(body))
|
||||
(HERE / "policies" / f"{space}_{actor}.written.json").write_text(json.dumps(body)) # survives the cascade
|
||||
print(f"egress policy {verb}d")
|
||||
'''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def kubectl_ate(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
"""A fake ``kubectl-ate`` in ``tmp_path`` (shared with the fake ax there); no settle wait."""
|
||||
path = tmp_path / "kubectl-ate"
|
||||
path.write_text(FAKE_KUBECTL_ATE.replace("{python}", sys.executable), encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
monkeypatch.setenv("OUSAST_KUBECTL_ATE", str(path))
|
||||
monkeypatch.setenv("OUSAST_EGRESS_SETTLE", "0")
|
||||
return path
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
"""ai-service-plane task 0: the per-task egress policy, its kubectl-ate calls, and the runner's dialled delivery.
|
||||
|
||||
Agent Substrate's egress gateway denies by default; :func:`egress.policy_for` must grant exactly the receiver (HTTP
|
||||
by name), the bound Workspaces' Git hosts and the bound Model's API host (TLS passthrough), and nothing a gateway
|
||||
would reject or that would intercept TLS: no IP address, no wildcard, no ``https`` rule.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.plane import egress, runner
|
||||
from openultrasast.plane.manifests import ManifestError, load_manifests, parse_manifest
|
||||
|
||||
MANIFESTS = """
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Model
|
||||
metadata:
|
||||
name: deepseek-flash
|
||||
annotations: {openultrasast.io/provider-extension: "true", openultrasast.io/egress-hosts: api.deepseek.com}
|
||||
spec: {provider: deepseek, model: deepseek-flash, secretKey: {name: deepseek, key: DEEPSEEK_API_KEY}}
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Workspace
|
||||
metadata: {name: cases}
|
||||
spec:
|
||||
git:
|
||||
- {name: a, repo: https://github.com/example-org/alpha}
|
||||
- {name: b, repo: https://github.com/example-org/beta.git}
|
||||
- {name: c, repo: "[email protected]:group/repo.git"}
|
||||
- {name: d, repo: https://10.0.0.7/mirror/repo.git}
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata: {name: verify, annotations: {openultrasast.io/model: deepseek-flash}}
|
||||
spec: {command: [verify], workspaces: [{name: cases, path: /workspace}]}
|
||||
---
|
||||
apiVersion: ax.io/v1alpha1
|
||||
kind: Task
|
||||
metadata: {name: facts}
|
||||
spec: {command: [repo-facts]}
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manifests(tmp_path: Path):
|
||||
path = tmp_path / "m.yaml"
|
||||
path.write_text(MANIFESTS, encoding="utf-8")
|
||||
return load_manifests([path])
|
||||
|
||||
|
||||
def hostnames(policy: dict) -> list[str]:
|
||||
return [h for rule in policy["rules"] for body in rule.values() for h in body["hostnames"]]
|
||||
|
||||
|
||||
def assert_tight(policy: dict) -> None:
|
||||
for rule in policy["rules"]:
|
||||
assert set(rule) <= {"http", "tlsPassthrough"}, f"only http and passthrough rules, got {rule}"
|
||||
body = next(iter(rule.values()))
|
||||
assert "all" not in body["ports"], "never every port"
|
||||
for host in hostnames(policy):
|
||||
assert "*" not in host, f"no wildcard: {host}"
|
||||
with pytest.raises(ValueError):
|
||||
ipaddress.ip_address(host)
|
||||
|
||||
|
||||
def test_model_free_task_gets_only_the_receiver(manifests) -> None:
|
||||
policy = egress.policy_for(manifests.tasks["facts"], [], None, egress.RECEIVER_HOST)
|
||||
assert policy == {"rules": [{"http": {"hostnames": [egress.RECEIVER_HOST], "ports": {"numbers": [80]}}}]}
|
||||
assert_tight(policy)
|
||||
|
||||
|
||||
def test_model_bound_task_with_git_workspaces(manifests) -> None:
|
||||
task = manifests.tasks["verify"]
|
||||
policy = egress.policy_for(task, [manifests.workspaces["cases"]], manifests.models["deepseek-flash"], egress.RECEIVER_HOST)
|
||||
assert policy["rules"] == [
|
||||
{"http": {"hostnames": [egress.RECEIVER_HOST], "ports": {"numbers": [80]}}},
|
||||
{"tlsPassthrough": {"hostnames": ["api.deepseek.com", "github.com"], "ports": {"numbers": [443]}}},
|
||||
], "one github.com for two repos; the ssh and IP-address repos add nothing"
|
||||
assert_tight(policy)
|
||||
|
||||
|
||||
def test_an_ip_receiver_is_not_granted(manifests) -> None:
|
||||
assert egress.policy_for(manifests.tasks["facts"], [], None, "127.0.0.1") == {"rules": []}
|
||||
|
||||
|
||||
def test_deepseek_model_manifest_names_the_endpoint_host() -> None:
|
||||
from openultrasast.model.endpoint import DEEPSEEK_BASE_URL
|
||||
|
||||
model = load_manifests([Path("plane/models/deepseek-flash.yaml")]).models["deepseek-flash"]
|
||||
assert model.egress_hosts == (DEEPSEEK_BASE_URL.removeprefix("https://"),)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hosts", ["*.deepseek.com", "10.1.2.3", "https://api.deepseek.com", "api.deepseek.com:443", "API.deepseek.com"])
|
||||
def test_egress_hosts_annotation_is_validated(hosts: str) -> None:
|
||||
doc = {
|
||||
"apiVersion": "ax.io/v1alpha1",
|
||||
"kind": "Model",
|
||||
"metadata": {"name": "m", "annotations": {"openultrasast.io/provider-extension": "true", "openultrasast.io/egress-hosts": hosts}},
|
||||
"spec": {"provider": "deepseek", "model": "x"},
|
||||
}
|
||||
with pytest.raises(ManifestError, match="egress-hosts"):
|
||||
parse_manifest(doc)
|
||||
|
||||
|
||||
# --- kubectl-ate ---------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def actor(tmp_path: Path, name: str, gone: bool = False) -> None:
|
||||
(tmp_path / "applied").mkdir(exist_ok=True)
|
||||
(tmp_path / "applied" / f"{name}.json").write_text(json.dumps({"gone": gone}), encoding="utf-8")
|
||||
|
||||
|
||||
def calls(tmp_path: Path) -> list[str]:
|
||||
return [" ".join(line.split()[2:4]) for line in (tmp_path / "ax.log").read_text().splitlines()]
|
||||
|
||||
|
||||
POLICY = {"rules": [{"http": {"hostnames": [egress.RECEIVER_HOST], "ports": {"numbers": [80]}}}]}
|
||||
|
||||
|
||||
def test_apply_creates_reads_back_and_delete_confirms_the_cascade(tmp_path: Path, kubectl_ate: Path) -> None:
|
||||
cli = egress.Egress(str(kubectl_ate), context="kind-test")
|
||||
actor(tmp_path, "t1")
|
||||
cli.apply("t1", "default", POLICY)
|
||||
stored = cli.get("t1", "default")
|
||||
assert stored is not None and egress.rules_of(stored) == egress.rules_of(POLICY)
|
||||
assert cli.delete("t1", "default") == "egress policy for default/t1 outlived its actor"
|
||||
actor(tmp_path, "t1", gone=True) # ax delete task: the actor, and with it the policy, is gone
|
||||
assert cli.delete("t1", "default") is None
|
||||
assert calls(tmp_path) == ["get t1", "create t1", "get t1", "get t1", "get t1", "get t1"]
|
||||
|
||||
|
||||
def test_apply_replaces_an_existing_policy(tmp_path: Path, kubectl_ate: Path) -> None:
|
||||
cli = egress.Egress(str(kubectl_ate))
|
||||
actor(tmp_path, "t2")
|
||||
cli.apply("t2", "default", {"rules": []})
|
||||
cli.apply("t2", "default", POLICY)
|
||||
assert calls(tmp_path) == ["get t2", "create t2", "get t2", "get t2", "update t2", "get t2"]
|
||||
assert json.loads((tmp_path / "policies" / "default_t2.json").read_text())["metadata"]["version"] == 2
|
||||
|
||||
|
||||
def test_apply_waits_for_the_actor_then_gives_up(tmp_path: Path, kubectl_ate: Path) -> None:
|
||||
cli = egress.Egress(str(kubectl_ate))
|
||||
with pytest.raises(RuntimeError, match="not found"):
|
||||
cli.apply("missing", "default", POLICY, timeout=0.2)
|
||||
timer = threading.Timer(0.3, actor, args=(tmp_path, "late"))
|
||||
timer.start()
|
||||
cli.apply("late", "default", POLICY, timeout=10)
|
||||
timer.join()
|
||||
|
||||
|
||||
def test_receiver_address_modes(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
|
||||
assert egress.receiver_address(4242) == ("http://127.0.0.1:4242/", None)
|
||||
monkeypatch.delenv("OUSAST_ARTIFACT_HOST")
|
||||
monkeypatch.setenv("OUSAST_ARTIFACT_DIAL", "172.19.0.1:80")
|
||||
assert egress.receiver_address(18090) == (f"http://{egress.RECEIVER_HOST}/", "172.19.0.1:80")
|
||||
|
||||
|
||||
# --- the runner dials the gateway address with the receiver's name as Host ----------------------------------------
|
||||
|
||||
|
||||
class _Seen(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
seen: list[tuple[str, str | None, bytes]]
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
server: _Seen
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
|
||||
self.server.seen.append((self.path, self.headers.get("Host"), body))
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None: # noqa: A002
|
||||
pass
|
||||
|
||||
|
||||
def test_delivery_dials_the_override_with_the_url_host() -> None:
|
||||
server = _Seen(("127.0.0.1", 0), _Handler)
|
||||
server.seen = []
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
dial = f"127.0.0.1:{server.server_address[1]}"
|
||||
url = f"http://{egress.RECEIVER_HOST}/in"
|
||||
assert runner.deliver(url, b"tar", {"Content-Type": "application/x-tar"}, backoff=0, dial=dial) == 200
|
||||
assert server.seen == [("/in", egress.RECEIVER_HOST, b"tar")]
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
@@ -156,7 +156,7 @@ apiVersion: ax.io/v1alpha1
|
||||
kind: Model
|
||||
metadata:
|
||||
name: deepseek-flash
|
||||
annotations: {openultrasast.io/provider-extension: "true"}
|
||||
annotations: {openultrasast.io/provider-extension: "true", openultrasast.io/egress-hosts: api.deepseek.com}
|
||||
spec:
|
||||
provider: deepseek
|
||||
model: deepseek-chat
|
||||
@@ -269,6 +269,8 @@ class Fake:
|
||||
out.append((stamp, Path(words[2]).parent.name))
|
||||
elif verb in ("delete", "resume") and words[:2] == [verb, "task"]:
|
||||
out.append((stamp, words[2]))
|
||||
elif verb == "egress" and words[0] == verb:
|
||||
out.append((stamp, f"{words[1]} {words[2]}"))
|
||||
elif verb in ("delivered", "rejected") and words[0] == verb:
|
||||
out.append((stamp, words[1]))
|
||||
return out
|
||||
@@ -313,7 +315,7 @@ class _RouterHandler(BaseHTTPRequestHandler):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Fake]:
|
||||
def fake(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, kubectl_ate: Path) -> Iterator[Fake]:
|
||||
monkeypatch.setenv("OUSAST_RESULTS", str(tmp_path / "results"))
|
||||
monkeypatch.setenv("OUSAST_ARTIFACT_HOST", "127.0.0.1")
|
||||
monkeypatch.setenv("OUSAST_POLL_SECONDS", "0.02")
|
||||
@@ -747,3 +749,34 @@ def test_doctor_treats_an_empty_namespace_as_not_ready(monkeypatch: pytest.Monke
|
||||
assert doctor._pods_ready("ns", "ctx") == (True, "2 pods ready")
|
||||
monkeypatch.setattr(doctor, "_sh", lambda *a: (True, "a-1 0/1 Running 0 1m"))
|
||||
assert doctor._pods_ready("ns", "ctx")[0] is False
|
||||
|
||||
|
||||
# --- egress (task 0: the per-task policy) --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_egress_policy_between_apply_and_resume_and_checked_after_delete(fake: Fake, tmp_path: Path) -> None:
|
||||
"""apply -> policy -> resume -> start -> delivery -> delete task -> the policy is gone with its actor."""
|
||||
assert reconciler.run(write_run(tmp_path, "eg"), ax=str(fake.ax)) == "done"
|
||||
lines = [" ".join(words) for _, words in fake.log()]
|
||||
|
||||
def first(prefix: str, after: int = 0) -> int:
|
||||
return next(i for i, line in enumerate(lines) if i >= after and line.startswith(prefix))
|
||||
|
||||
order = [first("apply -f " + str(reconciler.run_dir("eg") / "verify"))]
|
||||
steps = ("egress create eg-verify", "resume task eg-verify", "started eg-verify 202", "delivered verify", "delete task eg-verify")
|
||||
for prefix in (*steps, "egress get eg-verify"): # the last one: the check after the delete
|
||||
order.append(first(prefix, order[-1] + 1))
|
||||
assert order == sorted(order)
|
||||
assert not (tmp_path / "policies" / "default_eg-verify.json").exists(), "the policy went with its actor"
|
||||
written = json.loads((tmp_path / "policies" / "default_eg-verify.written.json").read_text())
|
||||
tls = {"hostnames": ["api.deepseek.com", "example.invalid"], "ports": {"numbers": [443]}}
|
||||
assert written["rules"] == [{"tlsPassthrough": tls}], "the Model host and the Git host, nothing else"
|
||||
facts = json.loads((tmp_path / "policies" / "default_eg-repo-facts.written.json").read_text())
|
||||
assert facts["rules"] == [{"tlsPassthrough": {"hostnames": ["example.invalid"], "ports": {"numbers": [443]}}}]
|
||||
|
||||
|
||||
def test_a_refused_egress_policy_fails_the_task_before_resume(fake: Fake, tmp_path: Path, kubectl_ate: Path) -> None:
|
||||
kubectl_ate.write_text("#!/bin/sh\necho 'Error: permission denied' >&2\nexit 1\n", encoding="utf-8")
|
||||
assert reconciler.run(write_run(tmp_path, "egfail"), ax=str(fake.ax)) == "failed"
|
||||
assert "permission denied" in state_of("egfail")["tasks"]["repo-facts"]["reason"]
|
||||
assert fake.events("resume") == [] and [n for _, n in fake.events("delete")] == ["egfail-repo-facts"]
|
||||
|
||||
@@ -244,6 +244,10 @@ def test_output_dir_is_delivered_as_a_tar_with_the_headers(
|
||||
assert headers["Content-Type"] == "application/x-tar"
|
||||
assert headers["X-Ousast-Task"] == "stub"
|
||||
assert headers["X-Ousast-Run"] == "validation-46"
|
||||
receiver.posts.clear()
|
||||
again = {"OUSAST_ARTIFACT_URL": receiver.url, "OUSAST_TASK": "facts", "OUSAST_STATE_DIR": str(tmp_path / "again-state")}
|
||||
code, _ = run_main(monkeypatch, tmp_path / "again", files_workspace_yaml(), **again)
|
||||
assert code == 0 and receiver.posts[0][0]["X-Ousast-Task"] == "facts", "the Run's task name, not the ax Task's"
|
||||
with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as archive:
|
||||
names = sorted(archive.getnames())
|
||||
summary = archive.extractfile("summary.json")
|
||||
|
||||
Reference in New Issue
Block a user