merge(engine): Kubernetes engine workers (Deployment from the image, S3 queue); leak-audit records carry digests, not repository names

The Joern leak-audit record named spent population v1/v2 repositories, which
the reserved-repository guard forbids outside benchmarks/independent/ (no v3
repository was involved: 0 matches). Records now carry counts and SHA-256
digests. With Codex.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

benchmarks/learn/engine_trace.py

@@ -1,4 +1,4 @@
-"""Resumable, serial labelled-function traces; host fetches pinned sources, Docker stays offline.
+"""Resumable labelled-function traces with Docker and Kubernetes execution lanes.
 
 Pair blob identities resolve through catalogs to real source commits. The analyzer
 is git archive HEAD src plus a hashed snapshot of this uncommitted tool.
@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import argparse
 import ast
+import copy
 import fcntl
 import hashlib
 import json
@@ -17,11 +18,15 @@ import shutil
 import subprocess
 import tempfile
 import textwrap
+import threading
 import time
 import tomllib
 from collections import Counter, defaultdict
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from pathlib import Path
 
+from engine_trace_k8s import IMAGE, Kubernetes
+from engine_trace_queue import Dispatcher
 from engine_trace_worker import EXCLUDED_DIRS, safe_path, unit_record, write_json
 
 from openultrasast.learn.examples import read_jsonl
@@ -39,6 +44,9 @@ DEFAULT_OUT = Path.home() / "ousast-results/plane/engine-trace"
 SNAPSHOT_FILES = (
     "benchmarks/learn/engine_trace.py",
     "benchmarks/learn/engine_trace_worker.py",
+    "benchmarks/learn/engine_trace_k8s.py",
+    "benchmarks/learn/engine_trace_queue.py",
+    "benchmarks/learn/engine_worker_service.py",
     "benchmarks/learn/engine_trace_parse.py",
     "src/openultrasast/model/taint.py",
     "src/openultrasast/cpg/queries/taint.sc",
@@ -611,32 +619,45 @@ def prepare_questions(pin, root, question_deadline):
 
 
 def run_pin(pin, source, provenance, inputs, out, args):
+    executor = getattr(args, "executor", "docker")
+    node = None
     with tempfile.TemporaryDirectory(prefix="trace-pin-", dir=out) as scratch:
         scratch = Path(scratch)
         checkout, output = scratch / "case", scratch / "output"
         output.mkdir()
         try:
-            pin = {**pin, "materialization": inputs.materialize(pin, checkout)}
+            with getattr(args, "input_lock", threading.Lock()):
+                pin = {**pin, "materialization": inputs.materialize(pin, checkout)}
             write_json(output / "pin.json", {**pin, "questions": prepare_questions(pin, checkout, args.question_deadline)})
             name = "ousast-trace-" + pin_name(pin)[:16]
-            command = docker_command(source, checkout, output, name, args.image, args.deadline, args.question_deadline)
             started = time.monotonic()
-            done = None
-            try:
-                done = subprocess.run(command, capture_output=True, text=True, timeout=args.deadline)
-                status, reason = "failed", f"container exit {done.returncode}: {done.stderr[-1000:]}"
-                (output / "container.log").write_text(done.stdout + "\n" + done.stderr)
-            except subprocess.TimeoutExpired:
+            if executor in {"k8s", "k8s-jobs"}:
+                done, node = args.kubernetes.execute(source, checkout, output, json.loads((output / "pin.json").read_text()))
+                status, reason = "failed", done.stderr
+                if done.returncode:
+                    (output / "container.log").write_text(reason)
+            elif executor == "queue":
+                done, node = args.dispatcher.execute(checkout, output, json.loads((output / "pin.json").read_text()), pin_name(pin)[:-5])
+                status, reason = "timeout", f"queue wait deadline {args.queue_timeout}s"
+            else:
+                command = docker_command(source, checkout, output, name, args.image, args.deadline, args.question_deadline)
+                started = time.monotonic()
                 done = None
-                status, reason = "timeout", f"container deadline {args.deadline}s"
-            finally:
-                # Never start the next pin unless this container is known to be gone.
                 try:
-                    removed = subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=20)
-                except subprocess.SubprocessError as exc:
-                    raise RuntimeError(f"cannot confirm container {name} stopped; refusing another launch") from exc
-                if done is None and removed.returncode != 0:
-                    raise RuntimeError(f"cannot remove timed-out container {name}; refusing another launch")
+                    done = subprocess.run(command, capture_output=True, text=True, timeout=args.deadline)
+                    status, reason = "failed", f"container exit {done.returncode}: {done.stderr[-1000:]}"
+                    (output / "container.log").write_text(done.stdout + "\n" + done.stderr)
+                except subprocess.TimeoutExpired:
+                    done = None
+                    status, reason = "timeout", f"container deadline {args.deadline}s"
+                finally:
+                    # Never start the next pin unless this container is known to be gone.
+                    try:
+                        removed = subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=20)
+                    except subprocess.SubprocessError as exc:
+                        raise RuntimeError(f"cannot confirm container {name} stopped; refusing another launch") from exc
+                    if done is None and removed.returncode != 0:
+                        raise RuntimeError(f"cannot remove timed-out container {name}; refusing another launch")
             result_path = output / "result.json"
             if not result_path.exists():
                 reason = f"container produced no result: {reason}"
@@ -651,12 +672,20 @@ def run_pin(pin, source, provenance, inputs, out, args):
                             status=status if unit["supported"] else "unsupported", reason=reason if unit["supported"] else unit["reason"]
                         )
                     record["units"].append(row)
-            record["container_exit"] = done.returncode if done else None
+            if executor != "queue":
+                record["container_exit"] = done.returncode if done else None
+            else:
+                record.setdefault("container_exit", None)
             if done is not None and done.returncode != 0:
                 for row in record["units"]:
                     if row["supported"]:
                         row.update(status="failed", reason=reason)
-            record.update(done=True, seconds=time.monotonic() - started, analyzer=provenance, image=args.image)
+            record.update(
+                done=True,
+                seconds=time.monotonic() - started,
+                analyzer=provenance,
+                image=record.get("image", args.image) if executor == "queue" else args.image,
+            )
             if (output / "container.log").exists():
                 shutil.copyfile(output / "container.log", out / pin_name(pin).replace(".json", ".log"))
         except (OSError, ValueError, subprocess.SubprocessError) as exc:
@@ -666,6 +695,10 @@ def run_pin(pin, source, provenance, inputs, out, args):
                 "analyzer": provenance,
                 "units": [unit_record(u, "failed" if u["supported"] else "unsupported", str(exc)) for u in pin["units"]],
             }
+        if executor in {"k8s", "k8s-jobs"}:
+            record.update(executor="k8s", node=node, image=args.image)
+        if executor == "queue":
+            record.update(executor="queue", worker=record.get("worker", node), pod=record.get("pod"))
         save_pin(out / pin_name(pin), record)
         print(
             json.dumps({"repo": pin["repo"], "pin": pin["pin"], "statuses": dict(Counter(u["status"] for u in record["units"]))}),
@@ -724,6 +757,67 @@ def summary(out):
     }
 
 
+def run_plan(plan, source, provenance, inputs, args):
+    """One task per fixed executor lane; progress is written only by the scheduler."""
+    lanes = ["k8s"] * (args.parallel - 1) + ["docker"] if args.executor == "mixed" else [args.executor] * args.parallel
+    args.input_lock = threading.Lock()
+    progress = {
+        "pins_done": 0,
+        "pins_total": len(plan),
+        "last_pin": None,
+        "statuses": {},
+        "lanes": dict(Counter(lanes)),
+        "pins_in_flight": 0,
+        "active_lanes": {},
+    }
+    counts = Counter()
+    pending = iter(plan)
+    active = {}
+    failures = False
+
+    def persist():
+        progress.update(pins_in_flight=len(active), active_lanes=dict(Counter(lane for _, lane in active.values())))
+        write_json(args.out / "progress.json", progress)
+
+    def execute(pin, lane):
+        if not any(u["supported"] for u in pin["units"]):
+            save_pin(
+                args.out / pin_name(pin), {**pin, "done": True, "units": [unit_record(u, "unsupported", u["reason"]) for u in pin["units"]]}
+            )
+            return False
+        options = copy.copy(args)
+        options.executor = lane
+        if lane == "docker":
+            options.image = getattr(args, "docker_image", args.image)
+        return run_pin(pin, source, provenance, inputs, args.out, options)
+
+    with ThreadPoolExecutor(max_workers=len(lanes)) as pool:
+
+        def launch(lane):
+            if not (args.out / "STOP").exists():
+                pin = next(pending, None)
+                if pin is not None:
+                    active[pool.submit(execute, pin, lane)] = (pin, lane)
+
+        for lane in lanes:
+            launch(lane)
+        persist()
+        while active:
+            completed, _ = wait(active, return_when=FIRST_COMPLETED)
+            for future in completed:
+                pin, lane = active.pop(future)
+                failures |= future.result()
+                result = json.loads((args.out / pin_name(pin)).read_text())
+                selected = {u["unit"] for u in pin["units"]}
+                counts.update(u["status"] for u in result["units"] if u["unit"] in selected)
+                progress.update(
+                    pins_done=progress["pins_done"] + 1, last_pin={"repo": pin["repo"], "pin": pin["pin"]}, statuses=dict(counts)
+                )
+                launch(lane)
+            persist()
+    return failures
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--units", type=Path, default=ROOT / "plane/experiments/exp-005-graph-slice.units.jsonl")
@@ -731,7 +825,15 @@ def main():
     parser.add_argument("--memory", type=Path, default=Path.home() / "ousast-results/plane/memory")
     parser.add_argument("--cache", type=Path, default=Path.home() / ".cache/openultrasast")
     parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
-    parser.add_argument("--image", default="openultrasast:dev")
+    parser.add_argument("--image", help="default: openultrasast:dev for Docker; public 2.0.1 image for k8s/mixed")
+    parser.add_argument("--executor", choices=("docker", "k8s", "k8s-jobs", "mixed", "queue"), default="docker")
+    parser.add_argument("--queue-status", action="store_true")
+    parser.add_argument("--queue-timeout", type=int, default=3600, help="total queue wait including execution, seconds")
+    parser.add_argument("--parallel", type=int, default=1, help="total pins in flight; mixed reserves one lane for Docker")
+    parser.add_argument("--kube-context")
+    parser.add_argument("--kube-namespace", default="ousast-engine")
+    parser.add_argument("--keep-jobs", action="store_true", help="retain Jobs, Secrets and staging objects for debugging")
+    parser.add_argument("--k8s-check", action="store_true")
     parser.add_argument("--deadline", type=int, default=900)
     parser.add_argument("--question-deadline", type=int, default=120)
     parser.add_argument("--limit", type=int, default=0)
@@ -745,8 +847,24 @@ def main():
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--summary", action="store_true")
     args = parser.parse_args()
+    if args.queue_status:
+        print(json.dumps(Dispatcher(args).status(), indent=2))
+        return 0
+    if args.queue_timeout <= 0:
+        parser.error("--queue-timeout must be positive")
+    if args.executor == "queue" and (not args.image or "@sha256:" not in args.image):
+        parser.error("queue requires --image with the deployed image digest")
+    if args.executor == "queue" and args.deadline > 900:
+        parser.error("queue deadline must be <= Deployment termination grace (900s)")
     if min(args.deadline, args.question_deadline) <= 0 or args.limit < 0:
         parser.error("deadlines must be positive; limit must be nonnegative")
+    if args.parallel < 1 or (args.executor == "mixed" and args.parallel < 2):
+        parser.error("--parallel must be positive; mixed needs at least 2 total lanes")
+    if (args.executor in {"k8s", "k8s-jobs", "mixed"} or args.k8s_check) and not args.kube_context:
+        parser.error("--kube-context is required for Kubernetes")
+    args.image = args.image or (IMAGE if args.executor != "docker" or args.k8s_check else "openultrasast:dev")
+    if args.k8s_check:
+        return Kubernetes(args).check()
     rerun_status = set(filter(None, args.rerun_status.split(",")))
     if rerun_status - set(STATUSES):
         parser.error("unknown --rerun-status value")
@@ -806,34 +924,23 @@ def main():
     except BlockingIOError:
         parser.error("another trace runner holds this results root")
     failures = False
-    if any(u["supported"] for pin in plan for u in pin["units"]):
+    args.docker_image = args.image
+    if args.executor in {"docker", "mixed"} and any(u["supported"] for pin in plan for u in pin["units"]):
         image = subprocess.run(
             ["docker", "image", "inspect", "--format", "{{.Id}}", args.image], capture_output=True, text=True, check=True
         )
-        args.image = image.stdout.strip()
-        if not args.image:
+        args.docker_image = image.stdout.strip()
+        if not args.docker_image:
             raise ValueError("Docker returned no local image identity")
+    if args.executor in {"k8s", "k8s-jobs", "mixed"} and any(u["supported"] for pin in plan for u in pin["units"]):
+        args.kubernetes = Kubernetes(args)
+        args.kubernetes.namespace()
+    if args.executor == "queue":
+        args.dispatcher = Dispatcher(args)
     with tempfile.TemporaryDirectory(prefix="trace-source-", dir=args.out) as scratch:
         source = Path(scratch) / "frozen"
-        provenance = freeze_source(source)
-        progress = {"pins_done": 0, "pins_total": len(plan), "last_pin": None, "statuses": {}}
-        counts = Counter()
-        write_json(args.out / "progress.json", progress)
-        for pin in plan:
-            if (args.out / "STOP").exists():
-                break
-            if not any(u["supported"] for u in pin["units"]):
-                save_pin(
-                    args.out / pin_name(pin),
-                    {**pin, "done": True, "units": [unit_record(u, "unsupported", u["reason"]) for u in pin["units"]]},
-                )
-            else:
-                failures |= run_pin(pin, source, provenance, inputs, args.out, args)
-            result = json.loads((args.out / pin_name(pin)).read_text())
-            selected = {u["unit"] for u in pin["units"]}
-            counts.update(u["status"] for u in result["units"] if u["unit"] in selected)
-            progress.update(pins_done=progress["pins_done"] + 1, last_pin={"repo": pin["repo"], "pin": pin["pin"]}, statuses=dict(counts))
-            write_json(args.out / "progress.json", progress)
+        provenance = {"image": args.image, "executor": "queue"} if args.executor == "queue" else freeze_source(source)
+        failures = run_plan(plan, source, provenance, inputs, args)
     return 2 if failures else 0
 
 

benchmarks/learn/engine_trace_k8s.py

@@ -0,0 +1,259 @@
+"""Host Kubernetes transport; the embedded bootstrap uses only the standard library."""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import re
+import subprocess
+import tarfile
+import time
+import uuid
+from datetime import timedelta
+
+from openultrasast.plane.memory import S3Store, open_store
+
+IMAGE = "ghcr.io/norandom/openultrasast:2.0.1"
+BOOTSTRAP = r"""
+import io, json, os, pathlib, subprocess, sys, tarfile, urllib.request
+
+def get(name):
+    with urllib.request.urlopen(os.environ[name], timeout=120) as response:
+        return response.read()
+
+def put(data):
+    request = urllib.request.Request(os.environ['RESULT_URL'], data=data, method='PUT')
+    with urllib.request.urlopen(request, timeout=120) as response:
+        response.read()
+
+try:
+    if os.environ.get('CHECK') == '1':
+        data = get('SOURCE_URL')
+        assert len(data) == 1024
+        subprocess.run([sys.executable, '-c', 'print(42)'], check=True)
+        put(data)
+    else:
+        for name, target in [('SOURCE_URL', '/case'), ('ANALYZER_URL', '/frozen')]:
+            pathlib.Path(target).mkdir(parents=True, exist_ok=True)
+            with tarfile.open(fileobj=io.BytesIO(get(name))) as archive:
+                archive.extractall(target, filter='data')
+        out = pathlib.Path('/out')
+        out.mkdir(exist_ok=True)
+        (out / 'pin.json').write_text(os.environ['PIN'])
+        env = dict(os.environ, PYTHONPATH='/frozen/src')
+        for key in ['SOURCE_URL', 'ANALYZER_URL', 'RESULT_URL']:
+            env.pop(key, None)
+        with (out / 'container.log').open('w') as log:
+            done = subprocess.run([sys.executable, '/frozen/benchmarks/learn/engine_trace_worker.py',
+                '--worker', '/out/pin.json', '--deadline', os.environ['DEADLINE'],
+                '--question-deadline', os.environ['QUESTION_DEADLINE']], env=env, stdout=log, stderr=log)
+        (out / 'exit.json').write_text(json.dumps(done.returncode))
+        buffer = io.BytesIO()
+        with tarfile.open(fileobj=buffer, mode='w') as archive:
+            for path in out.iterdir():
+                archive.add(path, arcname=path.name)
+        put(buffer.getvalue())
+        sys.exit(done.returncode)
+except Exception as exc:
+    # urllib exceptions can contain bearer URLs. Never print exception text.
+    print('engine bootstrap failed: ' + type(exc).__name__, file=sys.stderr)
+    sys.exit(1)
+"""
+
+
+def pack(path):
+    buffer = io.BytesIO()
+    with tarfile.open(fileobj=buffer, mode="w") as archive:
+        for child in sorted(path.iterdir()):
+            archive.add(child, arcname=child.name)
+    return buffer.getvalue()
+
+
+def manifests(name, run, args, urls, pin, check=False):
+    labels = {"engine-trace-run": run, "engine-trace-job": name, "engine-trace": "true"}
+    secret = {
+        "apiVersion": "v1",
+        "kind": "Secret",
+        "metadata": {"name": name, "namespace": args.kube_namespace, "labels": labels},
+        "type": "Opaque",
+        "stringData": urls,
+    }
+    env = [{"name": key, "valueFrom": {"secretKeyRef": {"name": name, "key": key}}} for key in urls]
+    env += [
+        {"name": key, "value": value}
+        for key, value in {
+            "PIN": json.dumps(pin),
+            "DEADLINE": str(args.deadline),
+            "QUESTION_DEADLINE": str(args.question_deadline),
+            "CHECK": "1" if check else "0",
+        }.items()
+    ]
+    job = {
+        "apiVersion": "batch/v1",
+        "kind": "Job",
+        "metadata": {"name": name, "namespace": args.kube_namespace, "labels": labels},
+        "spec": {
+            "activeDeadlineSeconds": args.deadline,
+            "backoffLimit": 0,
+            "ttlSecondsAfterFinished": 86400,
+            "template": {
+                "metadata": {"labels": labels},
+                "spec": {
+                    "restartPolicy": "Never",
+                    "automountServiceAccountToken": False,
+                    "securityContext": {"runAsUser": 1000, "runAsGroup": 1000, "fsGroup": 1000},
+                    "volumes": [{"name": name, "emptyDir": {}} for name in ("case", "frozen", "out")],
+                    "affinity": {
+                        "podAntiAffinity": {
+                            "requiredDuringSchedulingIgnoredDuringExecution": [
+                                {
+                                    "labelSelector": {"matchLabels": {"engine-trace": "true"}},
+                                    "topologyKey": "kubernetes.io/hostname",
+                                }
+                            ]
+                        }
+                    },
+                    "containers": [
+                        {
+                            "name": "engine",
+                            "image": args.image,
+                            "command": ["python", "-c", BOOTSTRAP],
+                            "env": env,
+                            "volumeMounts": [{"name": name, "mountPath": "/" + name} for name in ("case", "frozen", "out")],
+                            "resources": {"requests": {"memory": "3Gi", "cpu": "1"}, "limits": {"memory": "3Gi"}},
+                        }
+                    ],
+                },
+            },
+        },
+    }
+    return job, secret
+
+
+def redact(text):
+    return re.sub(r"https?://[^\s\"'<>]+", "[redacted URL]", text)
+
+
+class Kubernetes:
+    def __init__(self, args, store=None):
+        self.args = args
+        self.run = uuid.uuid4().hex
+        self.store = store if store is not None else open_store()
+        if not isinstance(self.store, S3Store):
+            raise ValueError("Kubernetes requires OUSAST_MEMORY=s3://bucket[/prefix]")
+
+    def kubectl(self, *args, manifest=None, check=True):
+        result = subprocess.run(
+            ["kubectl", "--context", self.args.kube_context, "--namespace", self.args.kube_namespace, *args],
+            input=json.dumps(manifest) if manifest is not None else None,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        if check and result.returncode:
+            # kubectl can echo submitted Secret data on errors; never expose stderr.
+            raise ValueError(f"kubectl {args[0]} failed (exit {result.returncode})")
+        return result
+
+    def namespace(self):
+        self.kubectl("get", "nodes", "-o", "json")
+        self.kubectl("apply", "-f", "-", manifest={"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": self.args.kube_namespace}})
+
+    def execute(self, source, checkout, output, pin, check=False):
+        name = "engine-" + uuid.uuid4().hex[:24]
+        pin_key = hashlib.sha256(json.dumps([pin.get("repo"), pin.get("pin")]).encode()).hexdigest()
+        prefix = f"engine-jobs/{self.run}/{pin_key}/"
+        keys = [prefix + file for file in ("source.tar", "analyzer.tar", "result.tar")]
+        expiry = timedelta(seconds=self.args.deadline + 900)
+        node = None
+        try:
+            self.store._put(keys[0], b"x" * 1024 if check else pack(checkout))
+            self.store._put(keys[1], b"" if check else pack(source))
+            urls = {
+                "SOURCE_URL": self.store.presign_get(keys[0], expiry),
+                "ANALYZER_URL": self.store.presign_get(keys[1], expiry),
+                "RESULT_URL": self.store.presign_put(keys[2], expiry),
+            }
+            job, secret = manifests(name, self.run, self.args, urls, pin, check)
+            # create avoids kubectl's last-applied annotation duplicating Secret values.
+            created = json.loads(self.kubectl("create", "-f", "-", "-o", "json", manifest=job).stdout)
+            secret["metadata"]["ownerReferences"] = [
+                {
+                    "apiVersion": "batch/v1",
+                    "kind": "Job",
+                    "name": name,
+                    "uid": created["metadata"]["uid"],
+                }
+            ]
+            self.kubectl("create", "-f", "-", manifest=secret)
+            end = time.monotonic() + self.args.deadline + 60
+            reason = "host deadline waiting for Job"
+            while time.monotonic() < end:
+                state = json.loads(self.kubectl("get", "job", name, "-o", "json").stdout).get("status", {})
+                pods = json.loads(self.kubectl("get", "pods", "-l", f"engine-trace-job={name}", "-o", "json").stdout)
+                node = next((p.get("spec", {}).get("nodeName") for p in pods.get("items", []) if p.get("spec", {}).get("nodeName")), node)
+                result = self.store._get(keys[2])
+                failed = next((c for c in state.get("conditions", []) if c.get("type") == "Failed" and c.get("status") == "True"), None)
+                if failed or state.get("failed"):
+                    reason = (failed or {}).get("reason", "Job failed") + ": " + (failed or {}).get("message", "")
+                    break
+                if state.get("succeeded") and result is not None:
+                    if check:
+                        if result[0] != b"x" * 1024:
+                            raise ValueError("round-trip bytes mismatch")
+                    else:
+                        with tarfile.open(fileobj=io.BytesIO(result[0])) as archive:
+                            archive.extractall(output, filter="data")
+                        if not (output / "result.json").is_file():
+                            raise ValueError("Job produced no result.json")
+                        record = json.loads((output / "result.json").read_text())
+                        if not record.get("done") or {u["unit"] for u in record["units"]} != {u["unit"] for u in pin["units"]}:
+                            raise ValueError("Job returned incomplete or mismatched units")
+                    return subprocess.CompletedProcess([], 0, "", ""), node
+                time.sleep(2)
+            logs = self.kubectl("logs", f"job/{name}", "--tail=40", check=False)
+            reason = redact(reason + "\n" + logs.stdout + logs.stderr)
+            return subprocess.CompletedProcess([], 1, "", reason), node
+        except Exception as exc:
+            reason = "Kubernetes transport failed: " + type(exc).__name__ + ": " + redact(str(exc))
+            try:
+                logs = self.kubectl("logs", f"job/{name}", "--tail=40", check=False)
+                reason += "\n" + redact(logs.stdout + logs.stderr)
+            except (OSError, ValueError, subprocess.SubprocessError):
+                pass
+            return subprocess.CompletedProcess([], 1, "", reason), node
+        finally:
+            if not self.args.keep_jobs:
+                # Attempt every cleanup even if an earlier deletion fails.
+                errors = []
+                for kind in ("job", "secret"):
+                    try:
+                        self.kubectl("delete", kind, name, "--ignore-not-found", "--wait=true", "--timeout=30s")
+                    except Exception as exc:
+                        errors.append(type(exc).__name__)
+                for key in keys:
+                    try:
+                        self.store._delete(key)
+                    except Exception as exc:
+                        errors.append(type(exc).__name__)
+                if errors:
+                    raise RuntimeError("engine Job cleanup failed: " + ", ".join(errors))
+
+    def check(self):
+        self.namespace()
+        nodes = json.loads(self.kubectl("get", "nodes", "-o", "json").stdout)
+        for node in nodes["items"]:
+            print(json.dumps({"node": node["metadata"]["name"], "allocatable_memory": node["status"]["allocatable"]["memory"]}))
+        done, node = self.execute(None, None, None, {}, check=True)
+        print(
+            json.dumps(
+                {
+                    "k8s_check": "passed" if done.returncode == 0 else "failed",
+                    "node": node,
+                    "reason": done.stderr,
+                    "round_trip_bytes": 1024 if done.returncode == 0 else 0,
+                }
+            )
+        )
+        return done.returncode

benchmarks/learn/engine_trace_queue.py

@@ -0,0 +1,50 @@
+"""Host queue transport: source and task metadata only, never analyzer code."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+import uuid
+
+from engine_worker_service import PENDING, client_from_env, pack, put, read, result_key, status
+
+
+class Dispatcher:
+    def __init__(self, args, client=None, *, pause=time.sleep, clock=time.monotonic):
+        self.client = client if client is not None else client_from_env()
+        self.run = uuid.uuid4().hex
+        self.args, self.pause, self.clock = args, pause, clock
+
+    def execute(self, checkout, output, pin, pin_id):
+        task_id = f"{self.run}-{pin_id}"
+        task = dict(
+            run=self.run,
+            id=pin_id,
+            task_id=task_id,
+            pin=pin,
+            input=f"engine-queue/inputs/{self.run}/{pin_id}/source.tar",
+            image=self.args.image,
+            deadline=self.args.deadline,
+            question_deadline=self.args.question_deadline,
+        )
+        try:
+            self.client.put(task["input"], pack(checkout), {})
+            put(self.client, PENDING + task_id, task)
+            end = self.clock() + self.args.queue_timeout
+            while self.clock() < end:
+                result = read(self.client, result_key(task))
+                if result is not None:
+                    (output / "result.json").write_text(json.dumps(result))
+                    self.client.delete(task["input"])
+                    return subprocess.CompletedProcess([], 0, "", ""), result.get("worker")
+                self.pause(2)
+            # Removing pending cancels work that has not been claimed; retain source for a live lease.
+            self.client.delete(PENDING + task_id)
+            return None, None
+        except Exception as exc:
+            # SDK errors can contain endpoint URLs or credentials. Do not persist their text.
+            raise OSError("queue transport failed") from exc
+
+    def status(self):
+        return status(self.client)

benchmarks/learn/engine_worker_service.py

@@ -0,0 +1,315 @@
+"""Image-resident queue workers. Requires strongly consistent S3 GET/PUT/LIST.
+
+Per-task Lamport bakery tickets use only ordinary object operations. A choosing
+marker precedes ticket selection; contenders wait for choosing peers and lower
+(ticket, worker) pairs. Late arrivals cannot displace a running owner. Leases
+assume bounded clock skew (less than MARGIN / 2) and a worker killed by its deadline.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import ctypes
+import io
+import json
+import logging
+import multiprocessing
+import os
+import signal
+import socket
+import tarfile
+import tempfile
+import threading
+import time
+import uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+from engine_trace_worker import unit_record, worker
+
+from openultrasast.plane.memory import S3Client, s3_settings
+
+PENDING = "engine-queue/pending/"
+CLAIMED = "engine-queue/claimed/"
+RESULTS = "engine-results/"
+MARGIN = 60
+LOG = logging.getLogger(__name__)
+
+
+def client_from_env():
+    settings = s3_settings(os.environ)
+    bucket = os.environ.get("S3_BUCKET")
+    if not bucket:
+        raise ValueError("S3_BUCKET is required")
+    return S3Client(bucket=bucket, **settings)
+
+
+def read(client, key):
+    value = client.get(key)
+    return json.loads(value[0]) if value else None
+
+
+def put(client, key, value):
+    client.put(key, json.dumps(value).encode(), {})
+
+
+def result_key(task):
+    return f"{RESULTS}{task['run']}/{task['id']}.json"
+
+
+def pack(root):
+    stream = io.BytesIO()
+    with tarfile.open(fileobj=stream, mode="w") as archive:
+        archive.add(root, arcname=".")
+    return stream.getvalue()
+
+
+def unpack(data, root):
+    with tarfile.open(fileobj=io.BytesIO(data)) as archive:
+        # Source snapshots need only regular files and directories, never links/devices.
+        for member in archive.getmembers():
+            target = (root / member.name).resolve()
+            if not target.is_relative_to(root.resolve()) or not (member.isfile() or member.isdir()):
+                raise ValueError("unsafe source archive")
+        archive.extractall(root, filter="data")
+
+
+class Queue:
+    def __init__(self, client, identity=None, *, now=time.time, pause=time.sleep, stop=None):
+        self.client = client
+        self.identity = identity or f"{socket.gethostname()}-{uuid.uuid4().hex}"
+        self.now, self.pause = now, pause
+        self.stop = stop or threading.Event()
+
+    def markers(self, task_id=None):
+        result = []
+        for key in self.client.keys(CLAIMED):
+            if task_id is None or key.rsplit("/", 1)[-1] == task_id:
+                marker = read(self.client, key)
+                if marker:
+                    result.append((key, marker))
+        return result
+
+    def tasks(self):
+        tasks = {}
+        for key in self.client.keys(PENDING):
+            task = read(self.client, key)
+            if task:
+                tasks[task["task_id"]] = task
+        # A dead worker may have removed pending already. Its task remains in the lease.
+        for _, marker in self.markers():
+            if marker["expires"] < self.now():
+                task = marker["task"]
+                tasks.setdefault(task["task_id"], task)
+        return list(tasks.values())
+
+    def claim(self, task):
+        if read(self.client, result_key(task)) is not None or self.stop.is_set():
+            return None
+        key = f"{CLAIMED}{self.identity}/{task['task_id']}"
+        marker = dict(task=task, ticket=0, expires=self.now() + task["deadline"] + MARGIN)
+        put(self.client, key, marker)
+        try:
+            live = [m for _, m in self.markers(task["task_id"]) if m["expires"] > self.now()]
+            marker["ticket"] = 1 + max((m["ticket"] for m in live), default=0)
+            put(self.client, key, marker)
+            while not self.stop.is_set():
+                # Do not start once the execution budget no longer fits within our lease.
+                if self.now() >= marker["expires"] - task["deadline"] - MARGIN / 2:
+                    break
+                peers = [(k, m) for k, m in self.markers(task["task_id"]) if k != key and m["expires"] > self.now()]
+                if any(m["ticket"] == 0 for _, m in peers):
+                    self.pause(0.1)
+                    continue
+                if any((m["ticket"], k) < (marker["ticket"], key) for k, m in peers):
+                    break  # loser backs off and can try another pending task
+                if read(self.client, key) != marker or read(self.client, result_key(task)) is not None:
+                    break
+                self.client.delete(PENDING + task["task_id"])
+                return key
+        except Exception:
+            self.client.delete(key)
+            raise
+        self.client.delete(key)
+        return None
+
+    def process_one(self, execute=None):
+        for task in self.tasks():
+            key = self.claim(task)
+            if key is None:
+                continue
+            try:
+                lease = read(self.client, key)
+                if lease is None:
+                    continue
+                budget = min(task["deadline"], lease["expires"] - self.now() - MARGIN / 2)
+                if budget <= 0:
+                    continue  # Leave the expired lease's task available to recovery.
+                result = (execute or execute_task)(self.client, {**task, "runtime_budget": budget})
+                result.update(executor="queue", worker=self.identity, pod=socket.gethostname(), image=os.environ.get("ENGINE_IMAGE", ""))
+                put(self.client, result_key(task), result)
+                # Publish the result before removing either the input or the claim.
+                self.client.delete(task["input"])
+                for old_key, old in self.markers(task["task_id"]):
+                    if old_key == key or old["expires"] < self.now():
+                        self.client.delete(old_key)
+                return True
+            except Exception:
+                # Keep the lease and task for recovery, without logging credential-bearing errors.
+                LOG.error("task failed; lease retained for recovery")
+                return False
+        return False
+
+
+def _child(pin, root, out, deadline, question_deadline):
+    os.setsid()
+    signal.signal(signal.SIGTERM, signal.SIG_DFL)
+    worker(pin, root, out, deadline, question_deadline)
+
+
+def adopt_orphans():
+    # Linux PR_SET_CHILD_SUBREAPER: escaped Joern sessions reparent here, not
+    # outside the service. There is exactly one task process tree per worker.
+    if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
+        raise OSError("cannot enable task descendant cleanup")
+
+
+def kill_children():
+    """Freeze parents before walking children, including new-session JVMs."""
+    seen = set()
+
+    def freeze(parent):
+        for entry in Path("/proc").iterdir():
+            if not entry.name.isdigit() or int(entry.name) in seen:
+                continue
+            try:
+                # comm can contain spaces or parentheses. Fields after it begin at state.
+                fields = (entry / "stat").read_text().rsplit(")", 1)[1].split()
+                if int(fields[1]) != parent:
+                    continue
+                pid = int(entry.name)
+                os.kill(pid, signal.SIGSTOP)
+                seen.add(pid)
+                freeze(pid)
+            except (FileNotFoundError, ProcessLookupError, PermissionError):
+                continue
+
+    # A child that exited during discovery can reparent descendants to this
+    # subreaper. Repeat until every remaining descendant is frozen.
+    while True:
+        count = len(seen)
+        freeze(os.getpid())
+        if len(seen) == count:
+            break
+    for pid in seen:
+        with contextlib.suppress(ProcessLookupError):
+            os.kill(pid, signal.SIGKILL)
+
+
+def execute_task(client, task):
+    adopt_orphans()
+    pin = task["pin"]
+    started = time.monotonic()
+    budget = task.get("runtime_budget", task["deadline"])
+    status, reason, code = "failed", "worker produced no result", None
+    with tempfile.TemporaryDirectory(prefix="engine-task-") as scratch:
+        root, out = Path(scratch) / "case", Path(scratch) / "result.json"
+        root.mkdir()
+        try:
+            if task["deadline"] > 900 or task["deadline"] <= 0:
+                raise ValueError("deadline exceeds deployment grace")
+            if task.get("image") != os.environ.get("ENGINE_IMAGE"):
+                raise ValueError("task image differs from deployed image")
+            source = client.get(task["input"])
+            if source is None:
+                raise ValueError("missing source")
+            unpack(source[0], root)
+            # The shipped worker also checks frontend input and graph output for every language.
+            files = [p for p in root.rglob("*") if p.is_file()]
+            if not files:
+                raise ValueError("empty source")
+            LOG.info("input proof: first file bytes=%d", len(files[0].read_bytes()))
+            if time.monotonic() - started >= budget:
+                status, reason = "timeout", "task input deadline"
+                raise TimeoutError(reason)
+            process = multiprocessing.get_context("fork").Process(target=_child, args=(pin, root, out, budget, task["question_deadline"]))
+            process.start()
+            try:
+                process.join(max(0, budget - (time.monotonic() - started)))
+                if process.is_alive():
+                    status, reason = "timeout", "task deadline"
+                else:
+                    reason = f"worker exit {process.exitcode}"
+                code = process.exitcode
+            finally:
+                # Joern creates separate sessions. Kill the whole descendant tree,
+                # not merely the Python wrapper's process group.
+                kill_children()
+                process.join()
+                # Reap adopted JVMs and their launchers after the direct child.
+                while True:
+                    try:
+                        os.waitpid(-1, 0)
+                    except ChildProcessError:
+                        break
+            record = json.loads(out.read_text()) if out.exists() else {**pin, "units": []}
+        except Exception:
+            record = {**pin, "units": []}
+            reason = "worker input or execution failed"
+        if not record.get("done") or code != 0:
+            previous = {u["unit"]: u for u in record["units"]}
+            record["units"] = [
+                previous[u["unit"]]
+                if code is None and previous.get(u["unit"], {}).get("status") in {"path", "asked-nothing"}
+                else unit_record(u, status if u["supported"] else "unsupported", reason if u["supported"] else u["reason"])
+                for u in pin["units"]
+            ]
+        record.update(done=True, container_exit=code, seconds=time.monotonic() - started)
+        return record
+
+
+def status(client):
+    return {
+        "pending": len(client.keys(PENDING)),
+        "claimed": len(client.keys(CLAIMED)),
+        "done": len([k for k in client.keys(RESULTS) if k.endswith(".json")]),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--port", type=int, default=8080)
+    args = parser.parse_args()
+    logging.basicConfig(level=logging.INFO)
+    stop = threading.Event()
+    signal.signal(signal.SIGTERM, lambda *_: stop.set())
+    signal.signal(signal.SIGINT, lambda *_: stop.set())
+    client = client_from_env()
+    queue = Queue(client, stop=stop)
+
+    class Health(BaseHTTPRequestHandler):
+        def do_GET(self):
+            self.send_response(200 if self.path == "/healthz" and not stop.is_set() else 503)
+            self.end_headers()
+
+        def log_message(self, *_):
+            pass
+
+    server = ThreadingHTTPServer(("0.0.0.0", args.port), Health)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    try:
+        while not stop.is_set():
+            try:
+                if not queue.process_one():
+                    stop.wait(2)
+            except Exception:
+                LOG.error("queue unavailable; retrying")
+                stop.wait(5)
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/learn/joern_leak_audit.py

@@ -135,9 +135,7 @@ def audit(units, results, *, expected_rows=501, expected_pairs=195):
         counts["stale"] += stale
         counts["unfinished"] += unfinished
         counts["skipped"] += stale or unfinished
-        files.append(
-            {"path": str(path), "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest(), "stale": stale, "unfinished": unfinished}
-        )
+        files.append((path.name, len(raw)))
         if stale or unfinished:
             continue
         counts["accepted_files"] += 1
@@ -156,8 +154,6 @@ def audit(units, results, *, expected_rows=501, expected_pairs=195):
                 **unit,
                 "source": row["source"],
                 "status": row["status"],
-                "repo": record["repo"],
-                "pin": record["pin"],
                 "pin_role": row.get("pin_role"),
                 **features(row),
             }
@@ -181,8 +177,9 @@ def audit(units, results, *, expected_rows=501, expected_pairs=195):
         "inputs": {
             **{k: counts[k] for k in ("accepted_files", "result_rows_read", "stale", "unfinished", "skipped")},
             "files_read": len(files),
-            "bytes_read": sum(f["bytes"] for f in files),
-            "files": files,
+            "bytes_read": sum(size for _, size in files),
+            # Hash a canonical inventory; filenames and paths may identify repositories.
+            "files_sha256": hashlib.sha256(json.dumps(sorted(files), separators=(",", ":")).encode()).hexdigest(),
         },
         "semantics": {
             "signals": "Shortest nonempty trace (first on ties); its sanitizer, guard and corresponding witness sink call target.",
@@ -195,7 +192,7 @@ def audit(units, results, *, expected_rows=501, expected_pairs=195):
             "limits": "Exploratory associations, not proof of leakage; running input snapshot, no multiple-testing correction.",
         },
         **scopes,
-        "units": rows,
+        "units": [{k: r[k] for k in ("unit", "pair", "family", "label", "fold", "source", "status", *SIGNALS, "sink_kind")} for r in rows],
     }
 
 
@@ -208,7 +205,7 @@ def main():
     raw = args.units.read_bytes()
     units = [json.loads(line) for line in raw.splitlines() if line.strip()]
     report = audit(units, args.results)
-    report["units_input"] = {"path": str(args.units), "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
+    report["units_input"] = {"bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
     overall, inputs = report["overall"], report["inputs"]
@@ -230,7 +227,7 @@ def main():
             return brief_metric(value)
 
         lines.append(f"{signal}: all={brief(overall['signals'][signal])}; both asked={brief(overall['both_asked_signals'][signal])}")
-    lines.append(f"Family/source metrics and per-file read evidence: {args.output}")
+    lines.append(f"Family/source metrics and input inventory digest: {args.output}")
     print("\n".join(lines))
 
 

benchmarks/learn/slice_leak_audit.py

@@ -141,7 +141,9 @@ def audit(store: MemoryStore, units: Sequence[Mapping[str, Any]], *, expected_ro
         "limitations": (
             "Flat IR: no dominance/guards, textual branch order, potential parameter inputs, conservative call returns; no alias analysis."
         ),
-        "units": rows,
+        "units": [
+            {k: r[k] for k in ("unit", "pair", "family", "label", "fold", "source", "status", "has_flow", "slice_length")} for r in rows
+        ],
     }