engine: worker Deployment from the image (S3 queue with leases, dispatcher --executor queue, engine-image workflow, scoped RustFS policy); merge main

Implemented with Codex. Cluster deploy pending the k3s kubeconfig and the scoped account.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

benchmarks/learn/engine_trace.py

@@ -26,6 +26,7 @@ from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from pathlib import Path
 
 from engine_trace_k8s import IMAGE, Kubernetes
+from engine_trace_queue import Dispatcher
 from engine_trace_worker import EXCLUDED_DIRS, safe_path, unit_record, write_json
 
 from openultrasast.learn.examples import read_jsonl
@@ -44,7 +45,9 @@ SNAPSHOT_FILES = (
     "benchmarks/learn/engine_trace.py",
     "benchmarks/learn/engine_trace_worker.py",
     "benchmarks/learn/engine_trace_k8s.py",
-    "src/openultrasast/model/trace.py",
+    "benchmarks/learn/engine_trace_queue.py",
+    "benchmarks/learn/engine_worker_service.py",
+    "benchmarks/learn/engine_trace_parse.py",
     "src/openultrasast/model/taint.py",
     "src/openultrasast/cpg/queries/taint.sc",
 )
@@ -628,11 +631,14 @@ def run_pin(pin, source, provenance, inputs, out, args):
             write_json(output / "pin.json", {**pin, "questions": prepare_questions(pin, checkout, args.question_deadline)})
             name = "ousast-trace-" + pin_name(pin)[:16]
             started = time.monotonic()
-            if executor == "k8s":
+            if executor in {"k8s", "k8s-jobs"}:
                 done, node = args.kubernetes.execute(source, checkout, output, json.loads((output / "pin.json").read_text()))
                 status, reason = "failed", done.stderr
                 if done.returncode:
                     (output / "container.log").write_text(reason)
+            elif executor == "queue":
+                done, node = args.dispatcher.execute(checkout, output, json.loads((output / "pin.json").read_text()), pin_name(pin)[:-5])
+                status, reason = "timeout", f"queue wait deadline {args.queue_timeout}s"
             else:
                 command = docker_command(source, checkout, output, name, args.image, args.deadline, args.question_deadline)
                 started = time.monotonic()
@@ -666,12 +672,20 @@ def run_pin(pin, source, provenance, inputs, out, args):
                             status=status if unit["supported"] else "unsupported", reason=reason if unit["supported"] else unit["reason"]
                         )
                     record["units"].append(row)
-            record["container_exit"] = done.returncode if done else None
+            if executor != "queue":
+                record["container_exit"] = done.returncode if done else None
+            else:
+                record.setdefault("container_exit", None)
             if done is not None and done.returncode != 0:
                 for row in record["units"]:
                     if row["supported"]:
                         row.update(status="failed", reason=reason)
-            record.update(done=True, seconds=time.monotonic() - started, analyzer=provenance, image=args.image)
+            record.update(
+                done=True,
+                seconds=time.monotonic() - started,
+                analyzer=provenance,
+                image=record.get("image", args.image) if executor == "queue" else args.image,
+            )
             if (output / "container.log").exists():
                 shutil.copyfile(output / "container.log", out / pin_name(pin).replace(".json", ".log"))
         except (OSError, ValueError, subprocess.SubprocessError) as exc:
@@ -681,8 +695,10 @@ def run_pin(pin, source, provenance, inputs, out, args):
                 "analyzer": provenance,
                 "units": [unit_record(u, "failed" if u["supported"] else "unsupported", str(exc)) for u in pin["units"]],
             }
-        if executor == "k8s":
+        if executor in {"k8s", "k8s-jobs"}:
             record.update(executor="k8s", node=node, image=args.image)
+        if executor == "queue":
+            record.update(executor="queue", worker=record.get("worker", node), pod=record.get("pod"))
         save_pin(out / pin_name(pin), record)
         print(
             json.dumps({"repo": pin["repo"], "pin": pin["pin"], "statuses": dict(Counter(u["status"] for u in record["units"]))}),
@@ -810,7 +826,9 @@ def main():
     parser.add_argument("--cache", type=Path, default=Path.home() / ".cache/openultrasast")
     parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
     parser.add_argument("--image", help="default: openultrasast:dev for Docker; public 2.0.1 image for k8s/mixed")
-    parser.add_argument("--executor", choices=("docker", "k8s", "mixed"), default="docker")
+    parser.add_argument("--executor", choices=("docker", "k8s", "k8s-jobs", "mixed", "queue"), default="docker")
+    parser.add_argument("--queue-status", action="store_true")
+    parser.add_argument("--queue-timeout", type=int, default=3600, help="total queue wait including execution, seconds")
     parser.add_argument("--parallel", type=int, default=1, help="total pins in flight; mixed reserves one lane for Docker")
     parser.add_argument("--kube-context")
     parser.add_argument("--kube-namespace", default="ousast-engine")
@@ -829,11 +847,20 @@ def main():
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--summary", action="store_true")
     args = parser.parse_args()
+    if args.queue_status:
+        print(json.dumps(Dispatcher(args).status(), indent=2))
+        return 0
+    if args.queue_timeout <= 0:
+        parser.error("--queue-timeout must be positive")
+    if args.executor == "queue" and (not args.image or "@sha256:" not in args.image):
+        parser.error("queue requires --image with the deployed image digest")
+    if args.executor == "queue" and args.deadline > 900:
+        parser.error("queue deadline must be <= Deployment termination grace (900s)")
     if min(args.deadline, args.question_deadline) <= 0 or args.limit < 0:
         parser.error("deadlines must be positive; limit must be nonnegative")
     if args.parallel < 1 or (args.executor == "mixed" and args.parallel < 2):
         parser.error("--parallel must be positive; mixed needs at least 2 total lanes")
-    if (args.executor != "docker" or args.k8s_check) and not args.kube_context:
+    if (args.executor in {"k8s", "k8s-jobs", "mixed"} or args.k8s_check) and not args.kube_context:
         parser.error("--kube-context is required for Kubernetes")
     args.image = args.image or (IMAGE if args.executor != "docker" or args.k8s_check else "openultrasast:dev")
     if args.k8s_check:
@@ -905,12 +932,14 @@ def main():
         args.docker_image = image.stdout.strip()
         if not args.docker_image:
             raise ValueError("Docker returned no local image identity")
-    if args.executor != "docker" and any(u["supported"] for pin in plan for u in pin["units"]):
+    if args.executor in {"k8s", "k8s-jobs", "mixed"} and any(u["supported"] for pin in plan for u in pin["units"]):
         args.kubernetes = Kubernetes(args)
         args.kubernetes.namespace()
+    if args.executor == "queue":
+        args.dispatcher = Dispatcher(args)
     with tempfile.TemporaryDirectory(prefix="trace-source-", dir=args.out) as scratch:
         source = Path(scratch) / "frozen"
-        provenance = freeze_source(source)
+        provenance = {"image": args.image, "executor": "queue"} if args.executor == "queue" else freeze_source(source)
         failures = run_plan(plan, source, provenance, inputs, args)
     return 2 if failures else 0
 

benchmarks/learn/engine_trace_parse.py

@@ -0,0 +1,51 @@
+"""Lossless parsing of the optional shortest-path trace, with explicit evidence limits."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from typing import Any
+
+_STEP = re.compile(r"^(.*):(-?\d+) \[([^\]]+)\] ([\s\S]*)$")
+
+
+def parse_trace(row: Mapping[str, Any]) -> dict[str, Any]:
+    """Preserve row-level discharge evidence; never guess which node sanitized a path.
+
+    The query emits basename-only files and truncated code. A bound can be off-path;
+    it remains in ``guard`` even when no trace step exactly matches it. Invalid steps
+    raise instead of turning an unreadable trace into an empty path.
+    """
+    raw = row.get("trace", [])
+    if not isinstance(raw, list):
+        raise ValueError("trace must be an array")
+    steps = []
+    bound = str(row.get("bound") or "")
+    for index, text in enumerate(raw):
+        match = _STEP.fullmatch(text) if isinstance(text, str) else None
+        if match is None:
+            raise ValueError(f"invalid trace step: {text!r}")
+        location, line, node_label, code = match.groups()
+        file, separator, method = location.partition(":")
+        roles = ["source"] if index == 0 else ["step"]
+        if index == len(raw) - 1:
+            roles = ["source", "sink"] if index == 0 else ["sink"]
+        if bound and code == bound:
+            roles.append("guard")
+        steps.append(
+            {
+                "file": file,
+                "method": method if separator else "?",
+                "line": int(line),
+                "node_label": node_label,
+                "code": code,
+                "roles": roles,
+                "raw": text,
+            }
+        )
+    return {
+        "steps": steps,
+        "sanitized": bool(row.get("sanitized", False)),
+        "sanitizer": {"on_path": bool(row.get("sanitized", False)), "step": None},
+        "guard": {"text": bound, "bounded": bool(row.get("bounded", False))},
+    }

benchmarks/learn/engine_trace_queue.py

@@ -0,0 +1,50 @@
+"""Host queue transport: source and task metadata only, never analyzer code."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+import uuid
+
+from engine_worker_service import PENDING, client_from_env, pack, put, read, result_key, status
+
+
+class Dispatcher:
+    def __init__(self, args, client=None, *, pause=time.sleep, clock=time.monotonic):
+        self.client = client if client is not None else client_from_env()
+        self.run = uuid.uuid4().hex
+        self.args, self.pause, self.clock = args, pause, clock
+
+    def execute(self, checkout, output, pin, pin_id):
+        task_id = f"{self.run}-{pin_id}"
+        task = dict(
+            run=self.run,
+            id=pin_id,
+            task_id=task_id,
+            pin=pin,
+            input=f"engine-queue/inputs/{self.run}/{pin_id}/source.tar",
+            image=self.args.image,
+            deadline=self.args.deadline,
+            question_deadline=self.args.question_deadline,
+        )
+        try:
+            self.client.put(task["input"], pack(checkout), {})
+            put(self.client, PENDING + task_id, task)
+            end = self.clock() + self.args.queue_timeout
+            while self.clock() < end:
+                result = read(self.client, result_key(task))
+                if result is not None:
+                    (output / "result.json").write_text(json.dumps(result))
+                    self.client.delete(task["input"])
+                    return subprocess.CompletedProcess([], 0, "", ""), result.get("worker")
+                self.pause(2)
+            # Removing pending cancels work that has not been claimed; retain source for a live lease.
+            self.client.delete(PENDING + task_id)
+            return None, None
+        except Exception as exc:
+            # SDK errors can contain endpoint URLs or credentials. Do not persist their text.
+            raise OSError("queue transport failed") from exc
+
+    def status(self):
+        return status(self.client)

benchmarks/learn/engine_trace_worker.py

@@ -19,8 +19,9 @@ import time
 from collections import Counter
 from pathlib import Path
 
+from engine_trace_parse import parse_trace
+
 from openultrasast.cpg.backend import DATAFLOW_OVERLAY, JoernBackend, _render
-from openultrasast.model.trace import parse_trace
 
 
 def write_json(path, value):

benchmarks/learn/engine_worker_service.py

@@ -0,0 +1,315 @@
+"""Image-resident queue workers. Requires strongly consistent S3 GET/PUT/LIST.
+
+Per-task Lamport bakery tickets use only ordinary object operations. A choosing
+marker precedes ticket selection; contenders wait for choosing peers and lower
+(ticket, worker) pairs. Late arrivals cannot displace a running owner. Leases
+assume bounded clock skew (less than MARGIN / 2) and a worker killed by its deadline.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import ctypes
+import io
+import json
+import logging
+import multiprocessing
+import os
+import signal
+import socket
+import tarfile
+import tempfile
+import threading
+import time
+import uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+from engine_trace_worker import unit_record, worker
+
+from openultrasast.plane.memory import S3Client, s3_settings
+
+PENDING = "engine-queue/pending/"
+CLAIMED = "engine-queue/claimed/"
+RESULTS = "engine-results/"
+MARGIN = 60
+LOG = logging.getLogger(__name__)
+
+
+def client_from_env():
+    settings = s3_settings(os.environ)
+    bucket = os.environ.get("S3_BUCKET")
+    if not bucket:
+        raise ValueError("S3_BUCKET is required")
+    return S3Client(bucket=bucket, **settings)
+
+
+def read(client, key):
+    value = client.get(key)
+    return json.loads(value[0]) if value else None
+
+
+def put(client, key, value):
+    client.put(key, json.dumps(value).encode(), {})
+
+
+def result_key(task):
+    return f"{RESULTS}{task['run']}/{task['id']}.json"
+
+
+def pack(root):
+    stream = io.BytesIO()
+    with tarfile.open(fileobj=stream, mode="w") as archive:
+        archive.add(root, arcname=".")
+    return stream.getvalue()
+
+
+def unpack(data, root):
+    with tarfile.open(fileobj=io.BytesIO(data)) as archive:
+        # Source snapshots need only regular files and directories, never links/devices.
+        for member in archive.getmembers():
+            target = (root / member.name).resolve()
+            if not target.is_relative_to(root.resolve()) or not (member.isfile() or member.isdir()):
+                raise ValueError("unsafe source archive")
+        archive.extractall(root, filter="data")
+
+
+class Queue:
+    def __init__(self, client, identity=None, *, now=time.time, pause=time.sleep, stop=None):
+        self.client = client
+        self.identity = identity or f"{socket.gethostname()}-{uuid.uuid4().hex}"
+        self.now, self.pause = now, pause
+        self.stop = stop or threading.Event()
+
+    def markers(self, task_id=None):
+        result = []
+        for key in self.client.keys(CLAIMED):
+            if task_id is None or key.rsplit("/", 1)[-1] == task_id:
+                marker = read(self.client, key)
+                if marker:
+                    result.append((key, marker))
+        return result
+
+    def tasks(self):
+        tasks = {}
+        for key in self.client.keys(PENDING):
+            task = read(self.client, key)
+            if task:
+                tasks[task["task_id"]] = task
+        # A dead worker may have removed pending already. Its task remains in the lease.
+        for _, marker in self.markers():
+            if marker["expires"] < self.now():
+                task = marker["task"]
+                tasks.setdefault(task["task_id"], task)
+        return list(tasks.values())
+
+    def claim(self, task):
+        if read(self.client, result_key(task)) is not None or self.stop.is_set():
+            return None
+        key = f"{CLAIMED}{self.identity}/{task['task_id']}"
+        marker = dict(task=task, ticket=0, expires=self.now() + task["deadline"] + MARGIN)
+        put(self.client, key, marker)
+        try:
+            live = [m for _, m in self.markers(task["task_id"]) if m["expires"] > self.now()]
+            marker["ticket"] = 1 + max((m["ticket"] for m in live), default=0)
+            put(self.client, key, marker)
+            while not self.stop.is_set():
+                # Do not start once the execution budget no longer fits within our lease.
+                if self.now() >= marker["expires"] - task["deadline"] - MARGIN / 2:
+                    break
+                peers = [(k, m) for k, m in self.markers(task["task_id"]) if k != key and m["expires"] > self.now()]
+                if any(m["ticket"] == 0 for _, m in peers):
+                    self.pause(0.1)
+                    continue
+                if any((m["ticket"], k) < (marker["ticket"], key) for k, m in peers):
+                    break  # loser backs off and can try another pending task
+                if read(self.client, key) != marker or read(self.client, result_key(task)) is not None:
+                    break
+                self.client.delete(PENDING + task["task_id"])
+                return key
+        except Exception:
+            self.client.delete(key)
+            raise
+        self.client.delete(key)
+        return None
+
+    def process_one(self, execute=None):
+        for task in self.tasks():
+            key = self.claim(task)
+            if key is None:
+                continue
+            try:
+                lease = read(self.client, key)
+                if lease is None:
+                    continue
+                budget = min(task["deadline"], lease["expires"] - self.now() - MARGIN / 2)
+                if budget <= 0:
+                    continue  # Leave the expired lease's task available to recovery.
+                result = (execute or execute_task)(self.client, {**task, "runtime_budget": budget})
+                result.update(executor="queue", worker=self.identity, pod=socket.gethostname(), image=os.environ.get("ENGINE_IMAGE", ""))
+                put(self.client, result_key(task), result)
+                # Publish the result before removing either the input or the claim.
+                self.client.delete(task["input"])
+                for old_key, old in self.markers(task["task_id"]):
+                    if old_key == key or old["expires"] < self.now():
+                        self.client.delete(old_key)
+                return True
+            except Exception:
+                # Keep the lease and task for recovery, without logging credential-bearing errors.
+                LOG.error("task failed; lease retained for recovery")
+                return False
+        return False
+
+
+def _child(pin, root, out, deadline, question_deadline):
+    os.setsid()
+    signal.signal(signal.SIGTERM, signal.SIG_DFL)
+    worker(pin, root, out, deadline, question_deadline)
+
+
+def adopt_orphans():
+    # Linux PR_SET_CHILD_SUBREAPER: escaped Joern sessions reparent here, not
+    # outside the service. There is exactly one task process tree per worker.
+    if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
+        raise OSError("cannot enable task descendant cleanup")
+
+
+def kill_children():
+    """Freeze parents before walking children, including new-session JVMs."""
+    seen = set()
+
+    def freeze(parent):
+        for entry in Path("/proc").iterdir():
+            if not entry.name.isdigit() or int(entry.name) in seen:
+                continue
+            try:
+                # comm can contain spaces or parentheses. Fields after it begin at state.
+                fields = (entry / "stat").read_text().rsplit(")", 1)[1].split()
+                if int(fields[1]) != parent:
+                    continue
+                pid = int(entry.name)
+                os.kill(pid, signal.SIGSTOP)
+                seen.add(pid)
+                freeze(pid)
+            except (FileNotFoundError, ProcessLookupError, PermissionError):
+                continue
+
+    # A child that exited during discovery can reparent descendants to this
+    # subreaper. Repeat until every remaining descendant is frozen.
+    while True:
+        count = len(seen)
+        freeze(os.getpid())
+        if len(seen) == count:
+            break
+    for pid in seen:
+        with contextlib.suppress(ProcessLookupError):
+            os.kill(pid, signal.SIGKILL)
+
+
+def execute_task(client, task):
+    adopt_orphans()
+    pin = task["pin"]
+    started = time.monotonic()
+    budget = task.get("runtime_budget", task["deadline"])
+    status, reason, code = "failed", "worker produced no result", None
+    with tempfile.TemporaryDirectory(prefix="engine-task-") as scratch:
+        root, out = Path(scratch) / "case", Path(scratch) / "result.json"
+        root.mkdir()
+        try:
+            if task["deadline"] > 900 or task["deadline"] <= 0:
+                raise ValueError("deadline exceeds deployment grace")
+            if task.get("image") != os.environ.get("ENGINE_IMAGE"):
+                raise ValueError("task image differs from deployed image")
+            source = client.get(task["input"])
+            if source is None:
+                raise ValueError("missing source")
+            unpack(source[0], root)
+            # The shipped worker also checks frontend input and graph output for every language.
+            files = [p for p in root.rglob("*") if p.is_file()]
+            if not files:
+                raise ValueError("empty source")
+            LOG.info("input proof: first file bytes=%d", len(files[0].read_bytes()))
+            if time.monotonic() - started >= budget:
+                status, reason = "timeout", "task input deadline"
+                raise TimeoutError(reason)
+            process = multiprocessing.get_context("fork").Process(target=_child, args=(pin, root, out, budget, task["question_deadline"]))
+            process.start()
+            try:
+                process.join(max(0, budget - (time.monotonic() - started)))
+                if process.is_alive():
+                    status, reason = "timeout", "task deadline"
+                else:
+                    reason = f"worker exit {process.exitcode}"
+                code = process.exitcode
+            finally:
+                # Joern creates separate sessions. Kill the whole descendant tree,
+                # not merely the Python wrapper's process group.
+                kill_children()
+                process.join()
+                # Reap adopted JVMs and their launchers after the direct child.
+                while True:
+                    try:
+                        os.waitpid(-1, 0)
+                    except ChildProcessError:
+                        break
+            record = json.loads(out.read_text()) if out.exists() else {**pin, "units": []}
+        except Exception:
+            record = {**pin, "units": []}
+            reason = "worker input or execution failed"
+        if not record.get("done") or code != 0:
+            previous = {u["unit"]: u for u in record["units"]}
+            record["units"] = [
+                previous[u["unit"]]
+                if code is None and previous.get(u["unit"], {}).get("status") in {"path", "asked-nothing"}
+                else unit_record(u, status if u["supported"] else "unsupported", reason if u["supported"] else u["reason"])
+                for u in pin["units"]
+            ]
+        record.update(done=True, container_exit=code, seconds=time.monotonic() - started)
+        return record
+
+
+def status(client):
+    return {
+        "pending": len(client.keys(PENDING)),
+        "claimed": len(client.keys(CLAIMED)),
+        "done": len([k for k in client.keys(RESULTS) if k.endswith(".json")]),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--port", type=int, default=8080)
+    args = parser.parse_args()
+    logging.basicConfig(level=logging.INFO)
+    stop = threading.Event()
+    signal.signal(signal.SIGTERM, lambda *_: stop.set())
+    signal.signal(signal.SIGINT, lambda *_: stop.set())
+    client = client_from_env()
+    queue = Queue(client, stop=stop)
+
+    class Health(BaseHTTPRequestHandler):
+        def do_GET(self):
+            self.send_response(200 if self.path == "/healthz" and not stop.is_set() else 503)
+            self.end_headers()
+
+        def log_message(self, *_):
+            pass
+
+    server = ThreadingHTTPServer(("0.0.0.0", args.port), Health)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    try:
+        while not stop.is_set():
+            try:
+                if not queue.process_one():
+                    stop.wait(2)
+            except Exception:
+                LOG.error("queue unavailable; retrying")
+                stop.wait(5)
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/learn/joern_leak_audit.py

@@ -0,0 +1,242 @@
+"""Offline audit of completed Joern results; never launches an analyzer or reads corpora.
+
+Missing results stay in coverage, not signal denominators. Observed non-path rows
+score zero for path features. Sink kinds are one-hot indicators, never ordinal
+category IDs. Both signed and best-direction metrics expose reversed shortcuts.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import hashlib
+import json
+import re
+from collections import Counter
+from pathlib import Path
+
+from slice_leak_audit import THRESHOLD, complete_pairs, metrics
+
+ROOT = Path(__file__).resolve().parents[2]
+UNITS = ROOT / "plane/experiments/exp-005-graph-slice.units.jsonl"
+RESULTS = Path.home() / "ousast-results/plane/engine-trace"
+OUTPUT = ROOT / "benchmarks/measurements/2026-10-04-joern-slice-leak-audit/record.json"
+STATUSES = ("path", "asked-nothing", "failed", "timeout", "unsupported", "missing")
+ASKED = {"path", "asked-nothing"}
+SIGNALS = ("has_path", "shortest_path_steps", "sanitized", "guard_present", "asked")
+ASYMMETRY = ("vulnerable_only", "fixed_only", "both", "neither", "not_both_asked")
+
+
+def materialization_version():
+    # Read the producer's literal without importing its execution dependencies.
+    tree = ast.parse(Path(__file__).with_name("engine_trace.py").read_bytes())
+    for node in tree.body:
+        if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "MATERIALIZATION_VERSION" for t in node.targets):
+            return ast.literal_eval(node.value)
+    raise ValueError("Producer has no materialization version")
+
+
+def features(row):
+    path = row["status"] == "path"
+    traces = row.get("traces", []) if path else []
+    candidates = [(i, t) for i, t in enumerate(traces) if t.get("steps")]
+    index, trace = min(candidates, key=lambda it: len(it[1]["steps"])) if candidates else (None, {})
+    witnesses = row.get("witness_rows", [])
+    witness = witnesses[index] if index is not None and index < len(witnesses) else {}
+    guard = trace.get("guard", {})
+    sink = witness.get("sinkKind")
+    if not sink:
+        # Retain only the call target: arguments, literals and code are not kinds.
+        match = re.search(r"([\w:$>.\\-]+)\s*\(", str(witness.get("sink", "")))
+        sink = match[1] if match else "unknown"
+    return {
+        "has_path": int(path),
+        "shortest_path_steps": len(trace["steps"]) if trace else None if path else 0,
+        "sanitized": int(bool(trace.get("sanitized", witness.get("sanitized", False)))) if trace else None if path else 0,
+        "guard_present": int(
+            bool(guard.get("text") or guard.get("bounded") or witness.get("bound") or witness.get("bounded"))
+            or any("guard" in s.get("roles", []) for s in trace.get("steps", []))
+        )
+        if trace
+        else None
+        if path
+        else 0,
+        "sink_kind": str(sink) if path else "no-path",
+        "asked": int(row["status"] in ASKED),
+    }
+
+
+def scored(rows, signal):
+    eligible = [r for r in rows if r[signal] is not None]
+    result = metrics(eligible, signal)
+    result["excluded_rows"] = len(rows) - len(eligible)
+    result["flagged"] = (result["pairs"] >= 10 and (result["best_direction_within_pair_accuracy"] or 0) >= THRESHOLD) or (
+        result["rows"] >= 10 and (result["best_direction_pooled_auc"] or 0) >= THRESHOLD
+    )
+    return result
+
+
+def signal_metrics(rows, kinds):
+    return {
+        **{s: scored(rows, s) for s in SIGNALS},
+        "sink_kind": {kind: scored([{**r, "is_kind": int(r["sink_kind"] == kind)} for r in rows], "is_kind") for kind in kinds},
+    }
+
+
+def scope_report(rows, kinds):
+    pairs = complete_pairs(rows)
+    asymmetry = Counter()
+    honest = []
+    for v, f in pairs:
+        if not (v["asked"] and f["asked"]):
+            asymmetry["not_both_asked"] += 1
+        else:
+            honest.extend((v, f))
+            asymmetry[
+                "both"
+                if v["has_path"] and f["has_path"]
+                else "vulnerable_only"
+                if v["has_path"]
+                else "fixed_only"
+                if f["has_path"]
+                else "neither"
+            ] += 1
+    observed = [r for r in rows if r["status"] != "missing"]
+    return {
+        "rows": len(rows),
+        "coverage": {s: sum(r["status"] == s for r in rows) for s in STATUSES},
+        "complete_pairs": len(pairs),
+        "pair_asymmetry": {s: asymmetry[s] for s in ASYMMETRY},
+        "both_asked_pairs": len(honest) // 2,
+        "signals": signal_metrics(observed, kinds),
+        "both_asked_signals": signal_metrics(honest, kinds),
+    }
+
+
+def audit(units, results, *, expected_rows=501, expected_pairs=195):
+    if len(units) != expected_rows or len({u["unit"] for u in units}) != expected_rows:
+        raise ValueError(f"Expected {expected_rows} unique units")
+    if len(complete_pairs(units)) != expected_pairs:
+        raise ValueError(f"Expected {expected_pairs} complete pairs")
+    if not results.is_dir():
+        raise ValueError(f"Results directory is unreadable: {results}")
+    version = materialization_version()
+    unit_index = {u["unit"]: u for u in units}
+    matched = {}
+    files = []
+    counts = Counter()
+    for path in sorted(results.glob("*.json")):
+        raw = path.read_bytes()
+        record = json.loads(raw)  # Corruption fails loudly; producer uses atomic replace.
+        if not isinstance(record, dict) or "repo" not in record or "pin" not in record:
+            continue
+        stale = record.get("materialization_version") != version
+        unfinished = record.get("done") is not True
+        counts["stale"] += stale
+        counts["unfinished"] += unfinished
+        counts["skipped"] += stale or unfinished
+        files.append(
+            {"path": str(path), "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest(), "stale": stale, "unfinished": unfinished}
+        )
+        if stale or unfinished:
+            continue
+        counts["accepted_files"] += 1
+        counts["result_rows_read"] += len(record["units"])
+        for row in record["units"]:
+            if row["unit"] not in unit_index:
+                continue
+            unit = unit_index[row["unit"]]
+            if any(row.get(k) != unit.get(k) for k in ("family", "label", "pair")):
+                raise ValueError(f"Result metadata mismatch: {row['unit']}")
+            if row["status"] not in STATUSES[:-1] or not row.get("source"):
+                raise ValueError(f"Invalid result status/source: {row['unit']}")
+            if row["unit"] in matched:
+                raise ValueError(f"Duplicate current result: {row['unit']}")
+            matched[row["unit"]] = {
+                **unit,
+                "source": row["source"],
+                "status": row["status"],
+                "repo": record["repo"],
+                "pin": record["pin"],
+                "pin_role": row.get("pin_role"),
+                **features(row),
+            }
+    rows = [
+        matched.get(u["unit"], {**u, "source": u.get("source", "unknown"), "status": "missing", **features({"status": "missing"})})
+        for u in units
+    ]
+    kinds = sorted({r["sink_kind"] for r in rows if r["status"] != "missing"})
+    scopes = {"overall": scope_report(rows, kinds)}
+    for field in ("family", "source"):
+        scopes[f"by_{field}"] = {
+            value: scope_report([r for r in rows if r[field] == value], kinds) for value in sorted({r[field] for r in rows})
+        }
+    return {
+        "cost_usd": 0,
+        "materialization_version": version,
+        "threshold": THRESHOLD,
+        "minimum_samples": 10,
+        "unit_rows_read": len(units),
+        "matched_rows": len(matched),
+        "inputs": {
+            **{k: counts[k] for k in ("accepted_files", "result_rows_read", "stale", "unfinished", "skipped")},
+            "files_read": len(files),
+            "bytes_read": sum(f["bytes"] for f in files),
+            "files": files,
+        },
+        "semantics": {
+            "signals": "Shortest nonempty trace (first on ties); its sanitizer, guard and corresponding witness sink call target.",
+            "missing": "Coverage only; absent source metadata is unknown. Skipped result units are never consumed.",
+            "non_path": "Observed non-path rows score zero; path rows lacking trace evidence have null trace features.",
+            "asked": "path or asked-nothing=1; failed, timeout, unsupported=0; missing excluded from metrics.",
+            "asymmetry": "Disjoint categories; not_both_asked takes precedence over path asymmetry.",
+            "flags": "Best of both directions, >=0.70 with >=10 pairs for pair accuracy or >=10 rows for AUC.",
+            "sink_kind": "One binary indicator per observed call target, including no-path/unknown; no ordinal encoding.",
+            "limits": "Exploratory associations, not proof of leakage; running input snapshot, no multiple-testing correction.",
+        },
+        **scopes,
+        "units": rows,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--results", type=Path, default=RESULTS)
+    parser.add_argument("--units", type=Path, default=UNITS)
+    parser.add_argument("--output", type=Path, default=OUTPUT)
+    args = parser.parse_args()
+    raw = args.units.read_bytes()
+    units = [json.loads(line) for line in raw.splitlines() if line.strip()]
+    report = audit(units, args.results)
+    report["units_input"] = {"path": str(args.units), "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
+    overall, inputs = report["overall"], report["inputs"]
+    lines = [
+        f"Joern leak audit: $0; materialization_version={report['materialization_version']}",
+        f"Units read: {len(units)}; bytes={len(raw)}; file={args.units}",
+        f"Result files read: {inputs['files_read']}; bytes={inputs['bytes_read']}; accepted={inputs['accepted_files']}",
+        f"Skipped: {inputs['skipped']} (stale={inputs['stale']}, unfinished={inputs['unfinished']}; counts overlap)",
+        f"Result rows read: {inputs['result_rows_read']}; matched={report['matched_rows']}",
+        f"Coverage: {overall['coverage']}",
+        f"Pairs: {overall['complete_pairs']}; both asked={overall['both_asked_pairs']}",
+        f"Asymmetry: {overall['pair_asymmetry']}",
+    ]
+    for signal in (*SIGNALS, "sink_kind"):
+
+        def brief(value, signal=signal):
+            if signal == "sink_kind":
+                return {k: brief_metric(m) for k, m in value.items()}
+            return brief_metric(value)
+
+        lines.append(f"{signal}: all={brief(overall['signals'][signal])}; both asked={brief(overall['both_asked_signals'][signal])}")
+    lines.append(f"Family/source metrics and per-file read evidence: {args.output}")
+    print("\n".join(lines))
+
+
+def brief_metric(metric):
+    return {k: metric[k] for k in ("rows", "pairs", "within_pair_accuracy", "pooled_auc", "flagged")}
+
+
+if __name__ == "__main__":
+    main()