engine: worker Deployment from the image (S3 queue with leases, dispatcher --executor queue, engine-image workflow, scoped RustFS policy); merge main

Implemented with Codex. Cluster deploy pending the k3s kubeconfig and the scoped account.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tests/test_advisory_fix_pairs.py

@@ -21,8 +21,9 @@ from openultrasast.pairs import load_pair_catalog
 
 ROOT = Path(__file__).resolve().parents[1]
 PAIRS = ROOT / "benchmarks" / "pairs"
-# batch 1 (2026-10-01) and batch 2 (2026-10-03, functions that survive the fix): one source each, same rules
-BATCHES = {"advisory-fixes": PAIRS / "advisory-fixes", "advisory-fixes-2": PAIRS / "advisory-fixes-2"}
+# batch 1 (2026-10-01), batches 2 (2026-10-03) and 3 (2026-10-04, functions that survive the fix): one source each,
+# same rules
+BATCHES = {name: PAIRS / name for name in ("advisory-fixes", "advisory-fixes-2", "advisory-fixes-3")}
 SHA = re.compile(r"[0-9a-f]{40}")
 PERMISSIVE_OR_COPYLEFT = re.compile(
     r"^(MIT|BSD-[23]-Clause|Apache-2\.0|ISC|Zlib|0BSD|Unlicense|MPL-[12]\.[01]"
@@ -106,10 +107,18 @@ def test_no_pair_repeats_a_repository_the_benchmarks_already_name(source_id: str
 PERMISSIVE = {"MIT", "BSD-2-Clause", "BSD-3-Clause", "Apache-2.0", "ISC", "MPL-2.0", "Unlicense"}
 
 
-def test_batch_two_is_permissive_and_dated_against_the_cutoff() -> None:
-    """Batch 2 publishes permissive licences only and records each advisory's publication against the 2026-06-01
+@pytest.mark.parametrize(("source_id", "batch"), [("advisory-fixes-2", "2026-10-03"), ("advisory-fixes-3", "2026-10-04")])
+def test_later_batches_are_permissive_and_dated_against_the_cutoff(source_id: str, batch: str) -> None:
+    """Batches 2 and 3 publish permissive licences only and record each advisory's publication against the 2026-06-01
     cutoff, so memorisation can be measured on the post-cutoff pairs."""
-    for item in _raw(BATCHES["advisory-fixes-2"]):
+    for item in _raw(BATCHES[source_id]):
         assert item["license"] in PERMISSIVE, item["name"]
-        assert item["batch"] == "2026-10-03", item["name"]
+        assert item["batch"] == batch, item["name"]
         assert item["post_cutoff"] is (item["published_at"][:10] >= "2026-06-01"), item["name"]
+
+
+def test_no_repository_repeats_across_batches() -> None:
+    seen: Counter[str] = Counter()
+    for folder in BATCHES.values():
+        seen.update({repo_name(item["repo"]) for item in _raw(folder)})
+    assert [repo for repo, n in seen.items() if n > 1] == []

tests/test_ax_lean.py

@@ -0,0 +1,130 @@
+"""Offline overlay contract and installer staging; no Kubernetes executable needed."""
+
+from __future__ import annotations
+
+import runpy
+from pathlib import Path
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+OVERLAY = ROOT / "ops/ax/substrate-kind-lean"
+DISABLED = {
+    "OTEL_TRACES_EXPORTER": "none",
+    "OTEL_METRICS_EXPORTER": "none",
+    "OTEL_LOGS_EXPORTER": "none",
+    "OTEL_SDK_DISABLED": "true",
+}
+
+
+def test_ax_lean_overlay_removes_observability_and_replaces_export_config() -> None:
+    overlay = yaml.safe_load((OVERLAY / "kustomization.yaml").read_text())
+    assert overlay["resources"] == ["../kind-upstream"]
+    deletes = [p for p in overlay["patches"] if "target" in p]
+    assert [p["target"] for p in deletes] == [
+        {"namespace": "otel-system"},
+        {"kind": "Namespace", "name": "otel-system"},
+        {"kind": "ClusterRole", "name": "ate-prometheus"},
+        {"kind": "RoleBinding", "name": "ate-prometheus", "namespace": "ate-system"},
+    ]
+    assert all(yaml.safe_load(p["patch"])["$patch"] == "delete" for p in deletes)
+    config = yaml.safe_load((OVERLAY / overlay["patches"][-1]["path"]).read_text())
+    assert config["metadata"] == {"name": "ate-otel-config", "namespace": "ate-system"}
+    # Whole-data replacement drops the upstream endpoint, interval and timeout,
+    # including any future keys that would otherwise survive a merge.
+    assert config["data"] == {"$patch": "replace", **DISABLED}
+
+
+def test_ax_staging_preserves_upstream_and_installer_prerequisites(tmp_path: Path) -> None:
+    prepare = runpy.run_path(str(ROOT / "ops/ax/prepare-substrate.py"))["prepare"]
+    source = tmp_path / "source"
+    kind = source / "manifests/ate-install/kind"
+    kind.mkdir(parents=True)
+    original = "resources: [otel-collector.yaml, prometheus.yaml, ate-otel-config.yaml]\n"
+    (kind / "kustomization.yaml").write_text(original)
+    (kind / "ate-otel-config.yaml").write_text("upstream config")
+    (kind / "postgres").mkdir()
+    (kind / "postgres/kustomization.yaml").write_text("resources: [../../postgres]\n")
+    (source / "go.mod").write_text("module test\n")
+    (source / "hack").mkdir()
+    (source / "hack/install-ate-kind.sh").write_text("upstream installer")
+    stage = tmp_path / "stage"
+    prepare(source, stage)
+    install = stage / "manifests/ate-install"
+    assert (install / "kind-upstream/kustomization.yaml").read_text() == original
+    assert (install / "kind/postgres/kustomization.yaml").read_text() == "resources: [../../postgres]\n"
+    assert (kind / "kustomization.yaml").read_text() == original
+    assert (kind / "ate-otel-config.yaml").read_text() == "upstream config"
+    assert (stage / "hack/install-ate-kind.sh").read_text() == "upstream installer"
+    assert (stage / "go.mod").is_symlink()
+    config = yaml.safe_load((install / "kind/ate-otel-config.yaml").read_text())
+    assert config["data"] == DISABLED  # direct installer apply must not contain $patch
+    overlay = yaml.safe_load((install / "kind/kustomization.yaml").read_text())
+    patch = yaml.safe_load((install / "kind" / overlay["patches"][-1]["path"]).read_text())
+    assert patch["data"] == {"$patch": "replace", **DISABLED}
+    with pytest.raises(ValueError, match="must be empty"):
+        prepare(source, stage)
+
+
+@pytest.mark.parametrize("enabled", [False, True])
+@pytest.mark.parametrize("existing", [False, True])
+def test_ax_reconciles_observability_on_ready_clusters(tmp_path: Path, enabled: bool, existing: bool) -> None:
+    """Execute the reconciliation shell with a function double, never real kubectl."""
+    import json
+    import os
+    import subprocess
+    import sys
+
+    script = (ROOT / "ops/ax/up.sh").read_text()
+    section = script.split("# Reconcile the choice", 1)[1].split('kubectl --context "$CTX" -n ate-system get pods', 1)[0]
+    section = "# Reconcile the choice" + section
+    source = tmp_path / "substrate/manifests/ate-install/kind"
+    source.mkdir(parents=True)
+    upstream_data = {"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4317", "OTEL_LOGS_EXPORTER": "otlp"}
+    (source / "ate-otel-config.yaml").write_text(yaml.safe_dump({"data": upstream_data}))
+    log = tmp_path / "calls"
+    consumer = {
+        "kind": "Deployment",
+        "metadata": {"name": "ate-controller"},
+        "spec": {"template": {"spec": {"containers": [{"envFrom": [{"configMapRef": {"name": "ate-otel-config"}}]}]}}},
+    }
+    nonconsumer = {"kind": "Deployment", "metadata": {"name": "unrelated"}, "spec": {"template": {"spec": {"containers": []}}}}
+    mock = """
+set -euo pipefail
+kubectl() {
+  printf '%s\\t' "$@" >> "$CALL_LOG"
+  printf '\\n' >> "$CALL_LOG"
+  case " $* " in
+    *" get deployment "*)
+      if [ "$EXISTING" = 1 ]; then printf 'deployment.apps/%s\\n' "$7"; fi ;;
+    *" get deployments,daemonsets "*) printf '%s' "$WORKLOADS" ;;
+  esac
+  return 0
+}
+"""
+    env = {
+        **os.environ,
+        "OBSERVABILITY": str(enabled).lower(),
+        "EXISTING": str(int(existing)),
+        "PY": sys.executable,
+        "CTX": "test-context",
+        "SRC": str(tmp_path),
+        "REPO_ROOT": str(ROOT),
+        "CALL_LOG": str(log),
+        "WORKLOADS": json.dumps({"items": [consumer, nonconsumer]}),
+    }
+    result = subprocess.run(["bash", "-c", mock + section], env=env, text=True, capture_output=True)
+    assert result.returncode == 0, result.stderr
+    calls = [line.rstrip("\t").split("\t") for line in log.read_text().splitlines()]
+    scales = [call for call in calls if "scale" in call]
+    assert len(scales) == (3 if existing and not enabled else 0)
+    assert all(call[-1] == "--replicas=0" for call in scales)
+    applies = [call for call in calls if "apply" in call]
+    assert len(applies) == (2 if enabled else 0)
+    config_patch = next(call[-1] for call in calls if "configmap" in call)
+    assert json.loads(config_patch) == [{"op": "replace", "path": "/data", "value": upstream_data if enabled else DISABLED}]
+    assert not any("deployment/unrelated" in call for call in calls)
+    assert any("deployment/ate-controller" in call for call in calls)
+    if not enabled:
+        assert "OUSAST_KIND_OBSERVABILITY=1" in result.stdout

tests/test_engine_queue.py

@@ -0,0 +1,290 @@
+"""Offline queue protocol, image imports and deployment contract controls."""
+
+from __future__ import annotations
+
+import importlib
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+with pytest.MonkeyPatch.context() as patch:
+    patch.syspath_prepend(str(ROOT / "benchmarks/learn"))
+    service = importlib.import_module("engine_worker_service")
+    transport = importlib.import_module("engine_trace_queue")
+
+
+class FakeS3:
+    def __init__(self):
+        self.objects = {}
+        self.lock = threading.Lock()
+        self.choosing = None
+
+    def get(self, key):
+        with self.lock:
+            return (self.objects[key], None) if key in self.objects else None
+
+    def put(self, key, data, labels):
+        with self.lock:
+            self.objects[key] = data
+        if self.choosing and key.startswith(service.CLAIMED) and json.loads(data)["ticket"] == 0:
+            self.choosing.wait(timeout=5)
+
+    def delete(self, key):
+        with self.lock:
+            self.objects.pop(key, None)
+
+    def keys(self, prefix):
+        with self.lock:
+            return sorted(k for k in self.objects if k.startswith(prefix))
+
+
+def task():
+    return dict(
+        run="run",
+        id="pin",
+        task_id="run-pin",
+        input="engine-queue/inputs/run/pin/source.tar",
+        deadline=10,
+        question_deadline=2,
+        pin={"repo": "owner/repo", "pin": "commit", "units": []},
+    )
+
+
+def test_engine_worker_racing_claimants_run_exactly_once():
+    store = FakeS3()
+    item = task()
+    service.put(store, service.PENDING + item["task_id"], item)
+    store.choosing = threading.Barrier(2)
+    called = []
+
+    def execute(client, entry):
+        called.append(entry["task_id"])
+        time.sleep(0.05)
+        return {**entry["pin"], "done": True, "instrument": {"bytes": 11}}
+
+    with ThreadPoolExecutor(2) as pool:
+        futures = [pool.submit(service.Queue(store, identity).process_one, execute) for identity in ["worker-b", "worker-a"]]
+        results = [f.result(timeout=10) for f in futures]
+    assert sum(results) == 1
+    assert called == ["run-pin"]
+    result = service.read(store, service.result_key(item))
+    assert result["worker"] in {"worker-a", "worker-b"}
+    assert not store.keys(service.CLAIMED)
+    assert not store.keys(service.PENDING)
+
+
+def test_engine_worker_expired_lease_recovered_without_pending():
+    store = FakeS3()
+    item = task()
+    service.put(store, "engine-queue/claimed/dead/run-pin", dict(task=item, ticket=1, expires=99))
+    queue = service.Queue(store, "new", now=lambda: 100)
+    assert queue.process_one(lambda *_: {**item["pin"], "done": True})
+    assert service.read(store, service.result_key(item))["worker"] == "new"
+    assert not store.keys(service.CLAIMED)
+
+
+def test_engine_worker_late_arrival_cannot_displace_running_owner():
+    store = FakeS3()
+    item = task()
+    first = service.Queue(store, "z-first")
+    key = first.claim(item)
+    assert key
+    stop = threading.Event()
+    later = service.Queue(store, "a-later", stop=stop)
+    with ThreadPoolExecutor(1) as pool:
+        future = pool.submit(later.claim, item)
+        assert future.result(timeout=3) is None
+        assert service.read(store, key) is not None
+        service.put(store, service.result_key(item), {"done": True})
+        store.delete(key)
+
+
+def test_engine_queue_dispatch_roundtrip_and_timeout(tmp_path):
+    store = FakeS3()
+    checkout, output = tmp_path / "case", tmp_path / "out"
+    checkout.mkdir()
+    output.mkdir()
+    (checkout / "app.py").write_text("print(1)\n")
+    args = SimpleNamespace(deadline=10, question_deadline=2, queue_timeout=2, image="image@sha256:abc")
+    unit = {"unit": "u", "supported": True, "reason": "", "family": "injection"}
+    pin = {"repo": "owner/repo", "pin": "sha", "units": [unit], "questions": []}
+
+    def execute(client, item):
+        unpacked = tmp_path / "unpacked"
+        unpacked.mkdir()
+        service.unpack(client.get(item["input"])[0], unpacked)
+        assert (unpacked / "app.py").read_bytes() == b"print(1)\n"
+        return {
+            **item["pin"],
+            "done": True,
+            "container_exit": 0,
+            "units": [service.unit_record(unit, "asked-nothing")],
+            "instrument": {"bytes": 9, "heap_mb": 2560},
+        }
+
+    queue = service.Queue(store, "pod-1")
+    dispatcher = transport.Dispatcher(args, store, pause=lambda _: queue.process_one(execute))
+    completed, worker = dispatcher.execute(checkout, output, pin, "pin")
+    assert completed.returncode == 0
+    assert worker == "pod-1"
+    result = json.loads((output / "result.json").read_text())
+    assert result["executor"] == "queue"
+    assert result["units"][0]["status"] == "asked-nothing"
+    assert result["instrument"]["heap_mb"] == 2560
+    assert dispatcher.status() == {"pending": 0, "claimed": 0, "done": 1}
+    assert not store.keys("engine-queue/inputs/")
+    ticks = iter([0, 3])
+    dispatcher = transport.Dispatcher(args, store, clock=lambda: next(ticks))
+    assert dispatcher.execute(checkout, output, pin, "other") == (None, None)
+    assert not store.keys(service.PENDING)
+
+
+def test_engine_worker_imports_without_yaml():
+    code = "import sys; sys.modules['yaml'] = None; import engine_worker_service; import engine_trace_worker"
+    done = subprocess.run(
+        [sys.executable, "-c", code],
+        capture_output=True,
+        text=True,
+        env={**os.environ, "PYTHONPATH": f"{ROOT / 'src'}:{ROOT / 'benchmarks/learn'}"},
+    )
+    assert done.returncode == 0, done.stderr
+
+
+def test_engine_k8s_render_contract():
+    base = ROOT / "ops/k8s/engine"
+    config = yaml.safe_load((base / "kustomization.yaml").read_text())
+    # Render this base's resource inclusion, namespace, image and replacement transforms offline.
+    resources = [yaml.safe_load((base / name).read_text()) for name in config["resources"]]
+    deployment = next(r for r in resources if r["kind"] == "Deployment")
+    deployment["metadata"]["namespace"] = config["namespace"]
+    image = config["images"][0]
+    pod = deployment["spec"]["template"]["spec"]
+    container = pod["containers"][0]
+    assert container["image"] == image["name"]
+    container["image"] = image["newName"] + "@" + image["digest"]
+    replacement = config["replacements"][0]
+    assert replacement["source"]["fieldPath"] == "spec.template.spec.containers.0.image"
+    assert replacement["targets"][0]["fieldPaths"] == ["spec.template.spec.containers.0.env.0.value"]
+    container["env"][0]["value"] = container["image"]
+    assert "@sha256:" in container["image"]
+    assert deployment["spec"]["replicas"] == 2
+    assert container["resources"] == {"requests": {"memory": "3Gi", "cpu": "1"}, "limits": {"memory": "3Gi"}}
+    affinity = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"][0]
+    assert affinity["topologyKey"] == "kubernetes.io/hostname"
+    assert affinity["labelSelector"]["matchLabels"] == deployment["spec"]["template"]["metadata"]["labels"]
+    assert pod["terminationGracePeriodSeconds"] == 900
+    assert container["envFrom"] == [{"secretRef": {"name": "ousast-engine-s3"}}]
+    for probe in ["readinessProbe", "livenessProbe"]:
+        assert container[probe]["httpGet"] == {"path": "/healthz", "port": "health"}
+    assert not pod["automountServiceAccountToken"]
+    assert pod["securityContext"]["runAsNonRoot"]
+    assert "hostPath" not in json.dumps(resources)
+    assert "privileged" not in json.dumps(resources)
+    assert all(r["kind"] != "Secret" for r in resources)
+    secret = yaml.safe_load((base / "secret.example.yaml").read_text())
+    assert secret["stringData"] == {}
+    assert "AWS_SECRET_ACCESS_KEY" not in json.dumps(resources)
+
+
+def test_engine_image_and_rustfs_policy():
+    dockerfile = (ROOT / "Dockerfile").read_text()
+    assert "COPY benchmarks ./benchmarks" in dockerfile
+    assert ".[semantic,s3]" in dockerfile
+    for name in ["engine_trace_worker.py", "engine_trace_parse.py", "engine_worker_service.py"]:
+        assert (ROOT / "benchmarks/learn" / name).stat().st_size > 0
+    doc = (ROOT / "docs/rustfs.md").read_text().split("## Scoped account for engine workers")[1]
+    policy = json.loads(doc.split("```json\n")[1].split("```")[0])
+    listing, objects = policy["Statement"]
+    assert listing["Action"] == "s3:ListBucket"
+    assert set(listing["Condition"]["StringLike"]["s3:prefix"]) == {
+        "engine-queue/",
+        "engine-queue/*",
+        "engine-results/",
+        "engine-results/*",
+    }
+    assert set(objects["Action"]) == {"s3:GetObject", "s3:PutObject", "s3:DeleteObject"}
+    assert set(objects["Resource"]) == {"arn:aws:s3:::sast-memory/engine-queue/*", "arn:aws:s3:::sast-memory/engine-results/*"}
+
+
+@pytest.mark.parametrize("hang", [False, True])
+def test_engine_worker_process_deadline_and_shipped_schema(tmp_path, monkeypatch, hang):
+    store = FakeS3()
+    item = task()
+    item["image"] = "image@sha256:abc"
+    item["deadline"] = 0.1 if hang else 5
+    item["pin"]["units"] = [{"unit": "u", "supported": True, "reason": ""}]
+    (tmp_path / "app.py").write_text("print(1)\n")
+    store.put(item["input"], service.pack(tmp_path), {})
+    monkeypatch.setenv("ENGINE_IMAGE", item["image"])
+
+    def fake_worker(pin, root, out, deadline, question_deadline):
+        assert (root / "app.py").read_bytes() == b"print(1)\n"
+        if hang:
+            time.sleep(10)
+        out.write_text(json.dumps({**pin, "done": True, "units": [service.unit_record(pin["units"][0], "asked-nothing")]}))
+
+    monkeypatch.setattr(service, "worker", fake_worker)
+    result = service.execute_task(store, item)
+    assert result["done"]
+    assert result["units"][0]["status"] == ("timeout" if hang else "asked-nothing")
+    assert result["container_exit"] == (None if hang else 0)
+    assert result["seconds"] < 5
+
+
+def test_engine_worker_stopping_does_not_claim():
+    store = FakeS3()
+    item = task()
+    service.put(store, service.PENDING + item["task_id"], item)
+    stop = threading.Event()
+    stop.set()
+    assert not service.Queue(store, stop=stop).process_one(lambda *_: pytest.fail("claimed after SIGTERM"))
+    assert store.keys(service.PENDING)
+    assert not store.keys(service.CLAIMED)
+
+
+def test_engine_worker_rejects_archive_links_and_escape(tmp_path):
+    import io
+    import tarfile
+
+    for name, kind in [("../escape", tarfile.REGTYPE), ("link", tarfile.SYMTYPE)]:
+        stream = io.BytesIO()
+        with tarfile.open(fileobj=stream, mode="w") as archive:
+            member = tarfile.TarInfo(name)
+            member.type = kind
+            member.linkname = "/etc/passwd"
+            archive.addfile(member)
+        with pytest.raises(ValueError, match="unsafe source archive"):
+            service.unpack(stream.getvalue(), tmp_path)
+
+
+def test_engine_worker_timeout_kills_separate_session_descendant(tmp_path, monkeypatch):
+    store = FakeS3()
+    item = task()
+    item.update(image="image@sha256:abc", deadline=0.5)
+    item["pin"]["units"] = [{"unit": "u", "supported": True, "reason": ""}]
+    (tmp_path / "app.py").write_text("print(1)\n")
+    store.put(item["input"], service.pack(tmp_path), {})
+    monkeypatch.setenv("ENGINE_IMAGE", item["image"])
+    pid_file = tmp_path / "escaped-pid"
+
+    def fake_worker(*_):
+        child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"], start_new_session=True)
+        pid_file.write_text(str(child.pid))
+        time.sleep(30)
+
+    monkeypatch.setattr(service, "worker", fake_worker)
+    result = service.execute_task(store, item)
+    assert result["units"][0]["status"] == "timeout"
+    assert pid_file.exists(), "instrument: fake worker never launched its descendant"
+    pid = int(pid_file.read_text())
+    assert not Path(f"/proc/{pid}").exists(), "escaped session survived the task deadline"

tests/test_engine_trace.py

@@ -14,7 +14,6 @@ import pytest
 from openultrasast.cpg.backend import BEGIN, END, CpgResult, JoernBackend
 from openultrasast.model.specs import taint_specs
 from openultrasast.model.taint import request_params
-from openultrasast.model.trace import parse_trace
 
 SCRIPT = Path(__file__).resolve().parents[1] / "benchmarks/learn/engine_trace.py"
 _spec = importlib.util.spec_from_file_location("engine_trace", SCRIPT)
@@ -22,6 +21,8 @@ assert _spec and _spec.loader
 trace = importlib.util.module_from_spec(_spec)
 with pytest.MonkeyPatch.context() as patch:
     patch.syspath_prepend(str(SCRIPT.parent))
+    from engine_trace_parse import parse_trace
+
     _spec.loader.exec_module(trace)
 worker = sys.modules["engine_trace_worker"]
 
@@ -304,6 +305,7 @@ def test_docker_command_is_serial_offline_frozen():
     assert cmd[-1] == "60"
     assert "/frozen/benchmarks/learn/engine_trace_worker.py" in cmd
     assert "benchmarks/learn/engine_trace_worker.py" in trace.SNAPSHOT_FILES
+    assert "benchmarks/learn/engine_trace_parse.py" in trace.SNAPSHOT_FILES
 
 
 def test_summary_separates_unanswered_pairs(tmp_path):
@@ -461,6 +463,8 @@ sys.modules['yaml'] = None
 sys.modules['boto3'] = None
 script = pathlib.Path(sys.argv[1])
 size = len(script.read_bytes())
+# Direct script execution in the image puts the sibling modules on sys.path.
+sys.path.insert(0, str(script.parent))
 runpy.run_path(str(script), run_name='trace_import_check')
 print(json.dumps({'bytes': size, 'modules': sorted(
     name for name in sys.modules if name == 'openultrasast' or name.startswith('openultrasast.')
@@ -486,7 +490,6 @@ def test_trace_container_transitive_import_boundary():
         "openultrasast.cpg.session",
         "openultrasast.model",
         "openultrasast.model.contracts",
-        "openultrasast.model.trace",
     ]
 
 
@@ -1397,3 +1400,55 @@ def test_k8s_check_roundtrip_and_retention(tmp_path, monkeypatch, capsys, keep):
     assert {"name": "CHECK", "value": "1"} in job["spec"]["template"]["spec"]["containers"][0]["env"]
     assert bool(data) is keep
     assert any("delete" in c for c in commands) is not keep
+
+
+@pytest.mark.parametrize("outcome", ["success", "timeout", "failure"])
+def test_engine_queue_pin_persistence_resume_and_progress(tmp_path, monkeypatch, outcome):
+    from engine_trace_queue import Dispatcher
+    from engine_worker_service import Queue
+    from test_engine_queue import FakeS3
+
+    store = FakeS3()
+    args = k8s_args(tmp_path, executor="queue", queue_timeout=2, image="image@sha256:abc")
+    plan = trace.make_plan([unit()], tmp_path)
+
+    class Inputs:
+        def materialize(self, pin, target):
+            target.mkdir()
+            (target / "app.py").write_text("def run(): pass")
+            return {"files": 1, "bytes": 15}
+
+    def execute(client, task):
+        return {
+            **task["pin"],
+            "done": True,
+            "container_exit": 0,
+            "units": [worker.unit_record(u, "asked-nothing") for u in task["pin"]["units"]],
+        }
+
+    queue = Queue(store, "pod-worker")
+    if outcome == "timeout":
+        ticks = iter([0, 3])
+        args.dispatcher = Dispatcher(args, store, clock=lambda: next(ticks))
+    else:
+        args.dispatcher = Dispatcher(args, store, pause=lambda _: queue.process_one(execute))
+    if outcome == "failure":
+
+        def fail(*_):
+            raise RuntimeError("credential-must-not-appear")
+
+        monkeypatch.setattr(store, "put", fail)
+    monkeypatch.setattr(trace, "prepare_questions", lambda *_: {"a": {}})
+    monkeypatch.setattr(subprocess, "run", lambda *_a, **_k: pytest.fail("queue must not launch Docker/kubectl"))
+    assert trace.run_plan(plan, None, {"image": args.image}, Inputs(), args) is (outcome != "success")
+    result = json.loads((tmp_path / trace.pin_name(plan[0])).read_text())
+    assert result["executor"] == "queue"
+    assert result["units"][0]["status"] == {"success": "asked-nothing", "timeout": "timeout", "failure": "failed"}[outcome]
+    assert "credential-must-not-appear" not in json.dumps(result)
+    if outcome == "success":
+        assert result["worker"] == "pod-worker"
+        assert trace.make_plan([unit()], tmp_path) == []
+    else:
+        assert trace.make_plan([unit()], tmp_path, rerun_status=("failed", "timeout"))
+    progress = json.loads((tmp_path / "progress.json").read_text())
+    assert progress["pins_done"] == 1 and progress["pins_in_flight"] == 0

tests/test_joern_leak_audit.py

@@ -0,0 +1,161 @@
+"""Synthetic, offline result snapshots; no analyzer, model or corpus access."""
+
+import importlib
+import json
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def audit_module(monkeypatch):
+    monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "benchmarks/learn"))
+    return importlib.import_module("joern_leak_audit")
+
+
+def units(count=12):
+    return [
+        {"unit": f"{pair}-{label}", "pair": str(pair), "label": label, "family": "injection", "group": "g", "fold": "f"}
+        for pair in range(count)
+        for label in (1, 0)
+    ]
+
+
+def write_result(root, module, rows, name="pin", *, done=True, version=None):
+    path = root / f"{name}.json"
+    path.write_text(
+        json.dumps(
+            {
+                "repo": "repo",
+                "pin": name,
+                "done": done,
+                "materialization_version": module.materialization_version() if version is None else version,
+                "units": rows,
+            }
+        )
+    )
+    return path
+
+
+def result(unit, status):
+    return {
+        **unit,
+        "source": "authored",
+        "status": status,
+        "pin_role": "vulnerable" if unit["label"] else "fixed",
+        "traces": [{"steps": [{"roles": ["source"]}, {"roles": ["sink"]}], "sanitized": False, "guard": {"text": "", "bounded": False}}]
+        if status == "path"
+        else [],
+        "witness_rows": [{"sink": "execute(x)"}] if status == "path" else [],
+    }
+
+
+def run(module, root, population):
+    return module.audit(population, root, expected_rows=len(population), expected_pairs=len(population) // 2)
+
+
+def test_skips_stale_unfinished_and_proves_read(audit_module, tmp_path):
+    population = units(2)
+    paths = [
+        write_result(tmp_path, audit_module, [result(population[0], "path")], "valid"),
+        write_result(tmp_path, audit_module, [result(population[1], "path")], "stale", version=-1),
+        write_result(tmp_path, audit_module, [result(population[2], "path")], "unfinished", done=False),
+        write_result(tmp_path, audit_module, [result(population[3], "path")], "both", version=-1, done=False),
+    ]
+    report = run(audit_module, tmp_path, population)
+    evidence = report["inputs"]
+    assert (evidence["stale"], evidence["unfinished"], evidence["skipped"]) == (2, 2, 3)
+    assert evidence["files_read"] == 4 and evidence["accepted_files"] == evidence["result_rows_read"] == 1
+    assert evidence["bytes_read"] == sum(p.stat().st_size for p in paths)
+    assert report["matched_rows"] == 1
+    assert report["overall"]["coverage"]["missing"] == 3
+    assert report["by_source"]["unknown"]["rows"] == 3
+    assert report["overall"]["signals"]["has_path"]["rows"] == 1
+    assert report["overall"]["both_asked_signals"]["has_path"]["pooled_auc"] is None
+
+
+def test_asymmetry_and_honest_denominator(audit_module, tmp_path):
+    population = units(6)
+    statuses = [
+        "path",
+        "asked-nothing",
+        "asked-nothing",
+        "path",
+        "path",
+        "path",
+        "asked-nothing",
+        "asked-nothing",
+        "path",
+        "failed",
+        "unsupported",
+        "timeout",
+    ]
+    write_result(tmp_path, audit_module, [result(u, s) for u, s in zip(population, statuses, strict=True)])
+    report = run(audit_module, tmp_path, population)
+    overall = report["overall"]
+    assert overall["pair_asymmetry"] == {"vulnerable_only": 1, "fixed_only": 1, "both": 1, "neither": 1, "not_both_asked": 2}
+    assert overall["both_asked_pairs"] == 4
+    assert overall["both_asked_signals"]["asked"]["within_pair_accuracy"] == 0.5
+    assert overall["coverage"] == {"path": 5, "asked-nothing": 4, "failed": 1, "timeout": 1, "unsupported": 1, "missing": 0}
+    assert report["by_family"]["injection"] == overall == report["by_source"]["authored"]
+
+
+@pytest.mark.parametrize("balanced", [False, True])
+def test_constructed_leak_and_balanced_set(audit_module, tmp_path, balanced):
+    population = units()
+    rows = [result(u, "path" if u["label"] == (int(u["pair"]) % 2 if balanced else 1) else "asked-nothing") for u in population]
+    write_result(tmp_path, audit_module, rows)
+    overall = run(audit_module, tmp_path, population)["overall"]
+    for denominator in ("signals", "both_asked_signals"):
+        signals = overall[denominator]
+        for name in ("has_path", "shortest_path_steps"):
+            assert signals[name]["flagged"] is not balanced
+            assert signals[name]["within_pair_accuracy"] == signals[name]["pooled_auc"] == (0.5 if balanced else 1)
+        assert signals["sink_kind"]["execute"]["flagged"] is not balanced
+        assert not signals["asked"]["flagged"]
+        assert not signals["sanitized"]["flagged"]
+        assert not signals["guard_present"]["flagged"]
+
+
+def test_minimum_samples_reverse_and_source_proxy(audit_module, tmp_path):
+    population = units(2)
+    write_result(tmp_path, audit_module, [result(u, "path" if not u["label"] else "asked-nothing") for u in population])
+    assert not run(audit_module, tmp_path, population)["overall"]["signals"]["has_path"]["flagged"]
+    population = units(12)
+    rows = []
+    for u in population:
+        row = result(u, "path" if not u["label"] else "unsupported")
+        row["source"] = "positive" if u["label"] else "negative"
+        rows.append(row)
+    write_result(tmp_path, audit_module, rows)
+    report = run(audit_module, tmp_path, population)
+    assert report["overall"]["signals"]["asked"]["flagged"]
+    assert report["overall"]["signals"]["asked"]["pooled_auc"] == 0
+    assert report["overall"]["both_asked_pairs"] == 0
+    assert report["by_source"]["positive"]["signals"]["asked"]["pooled_auc"] is None
+
+
+def test_shortest_trace_evidence_and_missing_trace(audit_module):
+    row = result(units(1)[0], "path")
+    row["traces"].insert(0, {"steps": [{}] * 5, "sanitized": False})
+    row["witness_rows"].insert(0, {"sink": "longer(x)"})
+    row["traces"][1].update(sanitized=True, guard={"bounded": True, "text": "off-path bound"})
+    features = audit_module.features(row)
+    assert (features["shortest_path_steps"], features["sanitized"], features["guard_present"], features["sink_kind"]) == (
+        2,
+        1,
+        1,
+        "execute",
+    )
+    row["traces"] = []
+    assert audit_module.features(row)["shortest_path_steps"] is None
+
+
+def test_bad_inputs_fail_loudly(audit_module, tmp_path):
+    population = units(1)
+    write_result(tmp_path, audit_module, [result(population[0], "path")] * 2)
+    with pytest.raises(ValueError, match="Duplicate"):
+        run(audit_module, tmp_path, population)
+    write_result(tmp_path, audit_module, [{**result(population[0], "path"), "label": 0}])
+    with pytest.raises(ValueError, match="metadata mismatch"):
+        run(audit_module, tmp_path, population)

tests/test_plane_profile.py

@@ -156,3 +156,27 @@ def test_profile_is_frozen_and_typed() -> None:
     assert isinstance(loaded, PlaneProfile)
     with pytest.raises(AttributeError):
         loaded.exec = "remote"  # type: ignore[misc]
+
+
+def test_kind_observability_defaults_off() -> None:
+    assert load_profile("kind", environ={}).kind_observability is False
+
+
+@pytest.mark.parametrize(("raw", "expected"), [("1", True), ("true", True), ("0", False), ("FALSE", False)])
+def test_kind_observability_environment_override(tmp_path: Path, raw: str, expected: bool) -> None:
+    path = write_profile(tmp_path, MINIMAL + f"kind_observability = {str(not expected).lower()}\n")
+    assert load_profile(str(path), environ={"OUSAST_KIND_OBSERVABILITY": raw}).kind_observability is expected
+
+
+def test_kind_observability_validation_and_print(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    path = write_profile(tmp_path, MINIMAL + 'kind_observability = "false"\n')
+    with pytest.raises(ProfileError, match="kind_observability must be a boolean"):
+        load_profile(str(path), environ={})
+    path = write_profile(tmp_path)
+    with pytest.raises(ProfileError, match="OUSAST_KIND_OBSERVABILITY must be"):
+        load_profile(str(path), environ={"OUSAST_KIND_OBSERVABILITY": "maybe"})
+    monkeypatch.setenv("OUSAST_KIND_OBSERVABILITY", "1")
+    assert profile_module.main(["--profile", str(path), "--print", "kind_observability"]) == 0
+    assert capsys.readouterr().out == "true\n"