merge(engine): Kubernetes engine workers (Deployment from the image, S3 queue); leak-audit records carry digests, not repository names

The Joern leak-audit record named spent population v1/v2 repositories, which
the reserved-repository guard forbids outside benchmarks/independent/ (no v3
repository was involved: 0 matches). Records now carry counts and SHA-256
digests. With Codex.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

.github/workflows/engine-image.yml

@@ -0,0 +1,47 @@
+name: Engine image
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - 'src/**'
+      - 'benchmarks/learn/engine_trace*.py'
+      - 'benchmarks/learn/engine_worker_service.py'
+      - 'Dockerfile'
+      - '.github/workflows/engine-image.yml'
+permissions:
+  contents: read
+  packages: write
+concurrency:
+  group: engine-image-${{ github.ref }}
+  cancel-in-progress: false
+jobs:
+  image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - id: names
+        env:
+          OWNER: ${{ github.repository_owner }}
+        run: |
+          echo "image=ghcr.io/${OWNER,,}/openultrasast" >> "$GITHUB_OUTPUT"
+          echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
+      - id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          tags: |
+            ${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
+            ${{ steps.names.outputs.image }}:main
+      - env:
+          IMAGE: ${{ steps.names.outputs.image }}
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"

Dockerfile

@@ -52,7 +52,7 @@ COPY benchmarks ./benchmarks
 # The core install is zero-dependency by design; `semantic` adds the tree-sitter grammars the candidate
 # enumerator uses. The LLM endpoint stays optional and is configured by environment, never baked in.
 RUN python3 -m venv /venv \
- && /venv/bin/pip install --no-cache-dir -e ".[semantic]"
+ && /venv/bin/pip install --no-cache-dir -e ".[semantic,s3]"
 ENV PATH="/venv/bin:${PATH}"
 
 # Analysis never needs root.

benchmarks/learn/engine_trace.py

@@ -1,4 +1,4 @@
-"""Resumable, serial labelled-function traces; host fetches pinned sources, Docker stays offline.
+"""Resumable labelled-function traces with Docker and Kubernetes execution lanes.
 
 Pair blob identities resolve through catalogs to real source commits. The analyzer
 is git archive HEAD src plus a hashed snapshot of this uncommitted tool.
@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import argparse
 import ast
+import copy
 import fcntl
 import hashlib
 import json
@@ -17,11 +18,15 @@ import shutil
 import subprocess
 import tempfile
 import textwrap
+import threading
 import time
 import tomllib
 from collections import Counter, defaultdict
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from pathlib import Path
 
+from engine_trace_k8s import IMAGE, Kubernetes
+from engine_trace_queue import Dispatcher
 from engine_trace_worker import EXCLUDED_DIRS, safe_path, unit_record, write_json
 
 from openultrasast.learn.examples import read_jsonl
@@ -39,6 +44,9 @@ DEFAULT_OUT = Path.home() / "ousast-results/plane/engine-trace"
 SNAPSHOT_FILES = (
     "benchmarks/learn/engine_trace.py",
     "benchmarks/learn/engine_trace_worker.py",
+    "benchmarks/learn/engine_trace_k8s.py",
+    "benchmarks/learn/engine_trace_queue.py",
+    "benchmarks/learn/engine_worker_service.py",
     "benchmarks/learn/engine_trace_parse.py",
     "src/openultrasast/model/taint.py",
     "src/openultrasast/cpg/queries/taint.sc",
@@ -611,32 +619,45 @@ def prepare_questions(pin, root, question_deadline):
 
 
 def run_pin(pin, source, provenance, inputs, out, args):
+    executor = getattr(args, "executor", "docker")
+    node = None
     with tempfile.TemporaryDirectory(prefix="trace-pin-", dir=out) as scratch:
         scratch = Path(scratch)
         checkout, output = scratch / "case", scratch / "output"
         output.mkdir()
         try:
-            pin = {**pin, "materialization": inputs.materialize(pin, checkout)}
+            with getattr(args, "input_lock", threading.Lock()):
+                pin = {**pin, "materialization": inputs.materialize(pin, checkout)}
             write_json(output / "pin.json", {**pin, "questions": prepare_questions(pin, checkout, args.question_deadline)})
             name = "ousast-trace-" + pin_name(pin)[:16]
-            command = docker_command(source, checkout, output, name, args.image, args.deadline, args.question_deadline)
             started = time.monotonic()
-            done = None
+            if executor in {"k8s", "k8s-jobs"}:
-            try:
+                done, node = args.kubernetes.execute(source, checkout, output, json.loads((output / "pin.json").read_text()))
-                done = subprocess.run(command, capture_output=True, text=True, timeout=args.deadline)
+                status, reason = "failed", done.stderr
-                status, reason = "failed", f"container exit {done.returncode}: {done.stderr[-1000:]}"
+                if done.returncode:
-                (output / "container.log").write_text(done.stdout + "\n" + done.stderr)
+                    (output / "container.log").write_text(reason)
-            except subprocess.TimeoutExpired:
+            elif executor == "queue":
+                done, node = args.dispatcher.execute(checkout, output, json.loads((output / "pin.json").read_text()), pin_name(pin)[:-5])
+                status, reason = "timeout", f"queue wait deadline {args.queue_timeout}s"
+            else:
+                command = docker_command(source, checkout, output, name, args.image, args.deadline, args.question_deadline)
+                started = time.monotonic()
                 done = None
-                status, reason = "timeout", f"container deadline {args.deadline}s"
-            finally:
-                # Never start the next pin unless this container is known to be gone.
                 try:
-                    removed = subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=20)
+                    done = subprocess.run(command, capture_output=True, text=True, timeout=args.deadline)
-                except subprocess.SubprocessError as exc:
+                    status, reason = "failed", f"container exit {done.returncode}: {done.stderr[-1000:]}"
-                    raise RuntimeError(f"cannot confirm container {name} stopped; refusing another launch") from exc
+                    (output / "container.log").write_text(done.stdout + "\n" + done.stderr)
-                if done is None and removed.returncode != 0:
+                except subprocess.TimeoutExpired:
-                    raise RuntimeError(f"cannot remove timed-out container {name}; refusing another launch")
+                    done = None
+                    status, reason = "timeout", f"container deadline {args.deadline}s"
+                finally:
+                    # Never start the next pin unless this container is known to be gone.
+                    try:
+                        removed = subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=20)
+                    except subprocess.SubprocessError as exc:
+                        raise RuntimeError(f"cannot confirm container {name} stopped; refusing another launch") from exc
+                    if done is None and removed.returncode != 0:
+                        raise RuntimeError(f"cannot remove timed-out container {name}; refusing another launch")
             result_path = output / "result.json"
             if not result_path.exists():
                 reason = f"container produced no result: {reason}"
@@ -651,12 +672,20 @@ def run_pin(pin, source, provenance, inputs, out, args):
                             status=status if unit["supported"] else "unsupported", reason=reason if unit["supported"] else unit["reason"]
                         )
                     record["units"].append(row)
-            record["container_exit"] = done.returncode if done else None
+            if executor != "queue":
+                record["container_exit"] = done.returncode if done else None
+            else:
+                record.setdefault("container_exit", None)
             if done is not None and done.returncode != 0:
                 for row in record["units"]:
                     if row["supported"]:
                         row.update(status="failed", reason=reason)
-            record.update(done=True, seconds=time.monotonic() - started, analyzer=provenance, image=args.image)
+            record.update(
+                done=True,
+                seconds=time.monotonic() - started,
+                analyzer=provenance,
+                image=record.get("image", args.image) if executor == "queue" else args.image,
+            )
             if (output / "container.log").exists():
                 shutil.copyfile(output / "container.log", out / pin_name(pin).replace(".json", ".log"))
         except (OSError, ValueError, subprocess.SubprocessError) as exc:
@@ -666,6 +695,10 @@ def run_pin(pin, source, provenance, inputs, out, args):
                 "analyzer": provenance,
                 "units": [unit_record(u, "failed" if u["supported"] else "unsupported", str(exc)) for u in pin["units"]],
             }
+        if executor in {"k8s", "k8s-jobs"}:
+            record.update(executor="k8s", node=node, image=args.image)
+        if executor == "queue":
+            record.update(executor="queue", worker=record.get("worker", node), pod=record.get("pod"))
         save_pin(out / pin_name(pin), record)
         print(
             json.dumps({"repo": pin["repo"], "pin": pin["pin"], "statuses": dict(Counter(u["status"] for u in record["units"]))}),
@@ -724,6 +757,67 @@ def summary(out):
     }
 
 
+def run_plan(plan, source, provenance, inputs, args):
+    """One task per fixed executor lane; progress is written only by the scheduler."""
+    lanes = ["k8s"] * (args.parallel - 1) + ["docker"] if args.executor == "mixed" else [args.executor] * args.parallel
+    args.input_lock = threading.Lock()
+    progress = {
+        "pins_done": 0,
+        "pins_total": len(plan),
+        "last_pin": None,
+        "statuses": {},
+        "lanes": dict(Counter(lanes)),
+        "pins_in_flight": 0,
+        "active_lanes": {},
+    }
+    counts = Counter()
+    pending = iter(plan)
+    active = {}
+    failures = False
+
+    def persist():
+        progress.update(pins_in_flight=len(active), active_lanes=dict(Counter(lane for _, lane in active.values())))
+        write_json(args.out / "progress.json", progress)
+
+    def execute(pin, lane):
+        if not any(u["supported"] for u in pin["units"]):
+            save_pin(
+                args.out / pin_name(pin), {**pin, "done": True, "units": [unit_record(u, "unsupported", u["reason"]) for u in pin["units"]]}
+            )
+            return False
+        options = copy.copy(args)
+        options.executor = lane
+        if lane == "docker":
+            options.image = getattr(args, "docker_image", args.image)
+        return run_pin(pin, source, provenance, inputs, args.out, options)
+
+    with ThreadPoolExecutor(max_workers=len(lanes)) as pool:
+
+        def launch(lane):
+            if not (args.out / "STOP").exists():
+                pin = next(pending, None)
+                if pin is not None:
+                    active[pool.submit(execute, pin, lane)] = (pin, lane)
+
+        for lane in lanes:
+            launch(lane)
+        persist()
+        while active:
+            completed, _ = wait(active, return_when=FIRST_COMPLETED)
+            for future in completed:
+                pin, lane = active.pop(future)
+                failures |= future.result()
+                result = json.loads((args.out / pin_name(pin)).read_text())
+                selected = {u["unit"] for u in pin["units"]}
+                counts.update(u["status"] for u in result["units"] if u["unit"] in selected)
+                progress.update(
+                    pins_done=progress["pins_done"] + 1, last_pin={"repo": pin["repo"], "pin": pin["pin"]}, statuses=dict(counts)
+                )
+                launch(lane)
+            persist()
+    return failures
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--units", type=Path, default=ROOT / "plane/experiments/exp-005-graph-slice.units.jsonl")
@@ -731,7 +825,15 @@ def main():
     parser.add_argument("--memory", type=Path, default=Path.home() / "ousast-results/plane/memory")
     parser.add_argument("--cache", type=Path, default=Path.home() / ".cache/openultrasast")
     parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
-    parser.add_argument("--image", default="openultrasast:dev")
+    parser.add_argument("--image", help="default: openultrasast:dev for Docker; public 2.0.1 image for k8s/mixed")
+    parser.add_argument("--executor", choices=("docker", "k8s", "k8s-jobs", "mixed", "queue"), default="docker")
+    parser.add_argument("--queue-status", action="store_true")
+    parser.add_argument("--queue-timeout", type=int, default=3600, help="total queue wait including execution, seconds")
+    parser.add_argument("--parallel", type=int, default=1, help="total pins in flight; mixed reserves one lane for Docker")
+    parser.add_argument("--kube-context")
+    parser.add_argument("--kube-namespace", default="ousast-engine")
+    parser.add_argument("--keep-jobs", action="store_true", help="retain Jobs, Secrets and staging objects for debugging")
+    parser.add_argument("--k8s-check", action="store_true")
     parser.add_argument("--deadline", type=int, default=900)
     parser.add_argument("--question-deadline", type=int, default=120)
     parser.add_argument("--limit", type=int, default=0)
@@ -745,8 +847,24 @@ def main():
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--summary", action="store_true")
     args = parser.parse_args()
+    if args.queue_status:
+        print(json.dumps(Dispatcher(args).status(), indent=2))
+        return 0
+    if args.queue_timeout <= 0:
+        parser.error("--queue-timeout must be positive")
+    if args.executor == "queue" and (not args.image or "@sha256:" not in args.image):
+        parser.error("queue requires --image with the deployed image digest")
+    if args.executor == "queue" and args.deadline > 900:
+        parser.error("queue deadline must be <= Deployment termination grace (900s)")
     if min(args.deadline, args.question_deadline) <= 0 or args.limit < 0:
         parser.error("deadlines must be positive; limit must be nonnegative")
+    if args.parallel < 1 or (args.executor == "mixed" and args.parallel < 2):
+        parser.error("--parallel must be positive; mixed needs at least 2 total lanes")
+    if (args.executor in {"k8s", "k8s-jobs", "mixed"} or args.k8s_check) and not args.kube_context:
+        parser.error("--kube-context is required for Kubernetes")
+    args.image = args.image or (IMAGE if args.executor != "docker" or args.k8s_check else "openultrasast:dev")
+    if args.k8s_check:
+        return Kubernetes(args).check()
     rerun_status = set(filter(None, args.rerun_status.split(",")))
     if rerun_status - set(STATUSES):
         parser.error("unknown --rerun-status value")
@@ -806,34 +924,23 @@ def main():
     except BlockingIOError:
         parser.error("another trace runner holds this results root")
     failures = False
-    if any(u["supported"] for pin in plan for u in pin["units"]):
+    args.docker_image = args.image
+    if args.executor in {"docker", "mixed"} and any(u["supported"] for pin in plan for u in pin["units"]):
         image = subprocess.run(
             ["docker", "image", "inspect", "--format", "{{.Id}}", args.image], capture_output=True, text=True, check=True
         )
-        args.image = image.stdout.strip()
+        args.docker_image = image.stdout.strip()
-        if not args.image:
+        if not args.docker_image:
             raise ValueError("Docker returned no local image identity")
+    if args.executor in {"k8s", "k8s-jobs", "mixed"} and any(u["supported"] for pin in plan for u in pin["units"]):
+        args.kubernetes = Kubernetes(args)
+        args.kubernetes.namespace()
+    if args.executor == "queue":
+        args.dispatcher = Dispatcher(args)
     with tempfile.TemporaryDirectory(prefix="trace-source-", dir=args.out) as scratch:
         source = Path(scratch) / "frozen"
-        provenance = freeze_source(source)
+        provenance = {"image": args.image, "executor": "queue"} if args.executor == "queue" else freeze_source(source)
-        progress = {"pins_done": 0, "pins_total": len(plan), "last_pin": None, "statuses": {}}
+        failures = run_plan(plan, source, provenance, inputs, args)
-        counts = Counter()
-        write_json(args.out / "progress.json", progress)
-        for pin in plan:
-            if (args.out / "STOP").exists():
-                break
-            if not any(u["supported"] for u in pin["units"]):
-                save_pin(
-                    args.out / pin_name(pin),
-                    {**pin, "done": True, "units": [unit_record(u, "unsupported", u["reason"]) for u in pin["units"]]},
-                )
-            else:
-                failures |= run_pin(pin, source, provenance, inputs, args.out, args)
-            result = json.loads((args.out / pin_name(pin)).read_text())
-            selected = {u["unit"] for u in pin["units"]}
-            counts.update(u["status"] for u in result["units"] if u["unit"] in selected)
-            progress.update(pins_done=progress["pins_done"] + 1, last_pin={"repo": pin["repo"], "pin": pin["pin"]}, statuses=dict(counts))
-            write_json(args.out / "progress.json", progress)
     return 2 if failures else 0
 
 

benchmarks/learn/engine_trace_k8s.py

@@ -0,0 +1,259 @@
+"""Host Kubernetes transport; the embedded bootstrap uses only the standard library."""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import re
+import subprocess
+import tarfile
+import time
+import uuid
+from datetime import timedelta
+
+from openultrasast.plane.memory import S3Store, open_store
+
+IMAGE = "ghcr.io/norandom/openultrasast:2.0.1"
+BOOTSTRAP = r"""
+import io, json, os, pathlib, subprocess, sys, tarfile, urllib.request
+
+def get(name):
+    with urllib.request.urlopen(os.environ[name], timeout=120) as response:
+        return response.read()
+
+def put(data):
+    request = urllib.request.Request(os.environ['RESULT_URL'], data=data, method='PUT')
+    with urllib.request.urlopen(request, timeout=120) as response:
+        response.read()
+
+try:
+    if os.environ.get('CHECK') == '1':
+        data = get('SOURCE_URL')
+        assert len(data) == 1024
+        subprocess.run([sys.executable, '-c', 'print(42)'], check=True)
+        put(data)
+    else:
+        for name, target in [('SOURCE_URL', '/case'), ('ANALYZER_URL', '/frozen')]:
+            pathlib.Path(target).mkdir(parents=True, exist_ok=True)
+            with tarfile.open(fileobj=io.BytesIO(get(name))) as archive:
+                archive.extractall(target, filter='data')
+        out = pathlib.Path('/out')
+        out.mkdir(exist_ok=True)
+        (out / 'pin.json').write_text(os.environ['PIN'])
+        env = dict(os.environ, PYTHONPATH='/frozen/src')
+        for key in ['SOURCE_URL', 'ANALYZER_URL', 'RESULT_URL']:
+            env.pop(key, None)
+        with (out / 'container.log').open('w') as log:
+            done = subprocess.run([sys.executable, '/frozen/benchmarks/learn/engine_trace_worker.py',
+                '--worker', '/out/pin.json', '--deadline', os.environ['DEADLINE'],
+                '--question-deadline', os.environ['QUESTION_DEADLINE']], env=env, stdout=log, stderr=log)
+        (out / 'exit.json').write_text(json.dumps(done.returncode))
+        buffer = io.BytesIO()
+        with tarfile.open(fileobj=buffer, mode='w') as archive:
+            for path in out.iterdir():
+                archive.add(path, arcname=path.name)
+        put(buffer.getvalue())
+        sys.exit(done.returncode)
+except Exception as exc:
+    # urllib exceptions can contain bearer URLs. Never print exception text.
+    print('engine bootstrap failed: ' + type(exc).__name__, file=sys.stderr)
+    sys.exit(1)
+"""
+
+
+def pack(path):
+    buffer = io.BytesIO()
+    with tarfile.open(fileobj=buffer, mode="w") as archive:
+        for child in sorted(path.iterdir()):
+            archive.add(child, arcname=child.name)
+    return buffer.getvalue()
+
+
+def manifests(name, run, args, urls, pin, check=False):
+    labels = {"engine-trace-run": run, "engine-trace-job": name, "engine-trace": "true"}
+    secret = {
+        "apiVersion": "v1",
+        "kind": "Secret",
+        "metadata": {"name": name, "namespace": args.kube_namespace, "labels": labels},
+        "type": "Opaque",
+        "stringData": urls,
+    }
+    env = [{"name": key, "valueFrom": {"secretKeyRef": {"name": name, "key": key}}} for key in urls]
+    env += [
+        {"name": key, "value": value}
+        for key, value in {
+            "PIN": json.dumps(pin),
+            "DEADLINE": str(args.deadline),
+            "QUESTION_DEADLINE": str(args.question_deadline),
+            "CHECK": "1" if check else "0",
+        }.items()
+    ]
+    job = {
+        "apiVersion": "batch/v1",
+        "kind": "Job",
+        "metadata": {"name": name, "namespace": args.kube_namespace, "labels": labels},
+        "spec": {
+            "activeDeadlineSeconds": args.deadline,
+            "backoffLimit": 0,
+            "ttlSecondsAfterFinished": 86400,
+            "template": {
+                "metadata": {"labels": labels},
+                "spec": {
+                    "restartPolicy": "Never",
+                    "automountServiceAccountToken": False,
+                    "securityContext": {"runAsUser": 1000, "runAsGroup": 1000, "fsGroup": 1000},
+                    "volumes": [{"name": name, "emptyDir": {}} for name in ("case", "frozen", "out")],
+                    "affinity": {
+                        "podAntiAffinity": {
+                            "requiredDuringSchedulingIgnoredDuringExecution": [
+                                {
+                                    "labelSelector": {"matchLabels": {"engine-trace": "true"}},
+                                    "topologyKey": "kubernetes.io/hostname",
+                                }
+                            ]
+                        }
+                    },
+                    "containers": [
+                        {
+                            "name": "engine",
+                            "image": args.image,
+                            "command": ["python", "-c", BOOTSTRAP],
+                            "env": env,
+                            "volumeMounts": [{"name": name, "mountPath": "/" + name} for name in ("case", "frozen", "out")],
+                            "resources": {"requests": {"memory": "3Gi", "cpu": "1"}, "limits": {"memory": "3Gi"}},
+                        }
+                    ],
+                },
+            },
+        },
+    }
+    return job, secret
+
+
+def redact(text):
+    return re.sub(r"https?://[^\s\"'<>]+", "[redacted URL]", text)
+
+
+class Kubernetes:
+    def __init__(self, args, store=None):
+        self.args = args
+        self.run = uuid.uuid4().hex
+        self.store = store if store is not None else open_store()
+        if not isinstance(self.store, S3Store):
+            raise ValueError("Kubernetes requires OUSAST_MEMORY=s3://bucket[/prefix]")
+
+    def kubectl(self, *args, manifest=None, check=True):
+        result = subprocess.run(
+            ["kubectl", "--context", self.args.kube_context, "--namespace", self.args.kube_namespace, *args],
+            input=json.dumps(manifest) if manifest is not None else None,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        if check and result.returncode:
+            # kubectl can echo submitted Secret data on errors; never expose stderr.
+            raise ValueError(f"kubectl {args[0]} failed (exit {result.returncode})")
+        return result
+
+    def namespace(self):
+        self.kubectl("get", "nodes", "-o", "json")
+        self.kubectl("apply", "-f", "-", manifest={"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": self.args.kube_namespace}})
+
+    def execute(self, source, checkout, output, pin, check=False):
+        name = "engine-" + uuid.uuid4().hex[:24]
+        pin_key = hashlib.sha256(json.dumps([pin.get("repo"), pin.get("pin")]).encode()).hexdigest()
+        prefix = f"engine-jobs/{self.run}/{pin_key}/"
+        keys = [prefix + file for file in ("source.tar", "analyzer.tar", "result.tar")]
+        expiry = timedelta(seconds=self.args.deadline + 900)
+        node = None
+        try:
+            self.store._put(keys[0], b"x" * 1024 if check else pack(checkout))
+            self.store._put(keys[1], b"" if check else pack(source))
+            urls = {
+                "SOURCE_URL": self.store.presign_get(keys[0], expiry),
+                "ANALYZER_URL": self.store.presign_get(keys[1], expiry),
+                "RESULT_URL": self.store.presign_put(keys[2], expiry),
+            }
+            job, secret = manifests(name, self.run, self.args, urls, pin, check)
+            # create avoids kubectl's last-applied annotation duplicating Secret values.
+            created = json.loads(self.kubectl("create", "-f", "-", "-o", "json", manifest=job).stdout)
+            secret["metadata"]["ownerReferences"] = [
+                {
+                    "apiVersion": "batch/v1",
+                    "kind": "Job",
+                    "name": name,
+                    "uid": created["metadata"]["uid"],
+                }
+            ]
+            self.kubectl("create", "-f", "-", manifest=secret)
+            end = time.monotonic() + self.args.deadline + 60
+            reason = "host deadline waiting for Job"
+            while time.monotonic() < end:
+                state = json.loads(self.kubectl("get", "job", name, "-o", "json").stdout).get("status", {})
+                pods = json.loads(self.kubectl("get", "pods", "-l", f"engine-trace-job={name}", "-o", "json").stdout)
+                node = next((p.get("spec", {}).get("nodeName") for p in pods.get("items", []) if p.get("spec", {}).get("nodeName")), node)
+                result = self.store._get(keys[2])
+                failed = next((c for c in state.get("conditions", []) if c.get("type") == "Failed" and c.get("status") == "True"), None)
+                if failed or state.get("failed"):
+                    reason = (failed or {}).get("reason", "Job failed") + ": " + (failed or {}).get("message", "")
+                    break
+                if state.get("succeeded") and result is not None:
+                    if check:
+                        if result[0] != b"x" * 1024:
+                            raise ValueError("round-trip bytes mismatch")
+                    else:
+                        with tarfile.open(fileobj=io.BytesIO(result[0])) as archive:
+                            archive.extractall(output, filter="data")
+                        if not (output / "result.json").is_file():
+                            raise ValueError("Job produced no result.json")
+                        record = json.loads((output / "result.json").read_text())
+                        if not record.get("done") or {u["unit"] for u in record["units"]} != {u["unit"] for u in pin["units"]}:
+                            raise ValueError("Job returned incomplete or mismatched units")
+                    return subprocess.CompletedProcess([], 0, "", ""), node
+                time.sleep(2)
+            logs = self.kubectl("logs", f"job/{name}", "--tail=40", check=False)
+            reason = redact(reason + "\n" + logs.stdout + logs.stderr)
+            return subprocess.CompletedProcess([], 1, "", reason), node
+        except Exception as exc:
+            reason = "Kubernetes transport failed: " + type(exc).__name__ + ": " + redact(str(exc))
+            try:
+                logs = self.kubectl("logs", f"job/{name}", "--tail=40", check=False)
+                reason += "\n" + redact(logs.stdout + logs.stderr)
+            except (OSError, ValueError, subprocess.SubprocessError):
+                pass
+            return subprocess.CompletedProcess([], 1, "", reason), node
+        finally:
+            if not self.args.keep_jobs:
+                # Attempt every cleanup even if an earlier deletion fails.
+                errors = []
+                for kind in ("job", "secret"):
+                    try:
+                        self.kubectl("delete", kind, name, "--ignore-not-found", "--wait=true", "--timeout=30s")
+                    except Exception as exc:
+                        errors.append(type(exc).__name__)
+                for key in keys:
+                    try:
+                        self.store._delete(key)
+                    except Exception as exc:
+                        errors.append(type(exc).__name__)
+                if errors:
+                    raise RuntimeError("engine Job cleanup failed: " + ", ".join(errors))
+
+    def check(self):
+        self.namespace()
+        nodes = json.loads(self.kubectl("get", "nodes", "-o", "json").stdout)
+        for node in nodes["items"]:
+            print(json.dumps({"node": node["metadata"]["name"], "allocatable_memory": node["status"]["allocatable"]["memory"]}))
+        done, node = self.execute(None, None, None, {}, check=True)
+        print(
+            json.dumps(
+                {
+                    "k8s_check": "passed" if done.returncode == 0 else "failed",
+                    "node": node,
+                    "reason": done.stderr,
+                    "round_trip_bytes": 1024 if done.returncode == 0 else 0,
+                }
+            )
+        )
+        return done.returncode

benchmarks/learn/engine_trace_queue.py

@@ -0,0 +1,50 @@
+"""Host queue transport: source and task metadata only, never analyzer code."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+import uuid
+
+from engine_worker_service import PENDING, client_from_env, pack, put, read, result_key, status
+
+
+class Dispatcher:
+    def __init__(self, args, client=None, *, pause=time.sleep, clock=time.monotonic):
+        self.client = client if client is not None else client_from_env()
+        self.run = uuid.uuid4().hex
+        self.args, self.pause, self.clock = args, pause, clock
+
+    def execute(self, checkout, output, pin, pin_id):
+        task_id = f"{self.run}-{pin_id}"
+        task = dict(
+            run=self.run,
+            id=pin_id,
+            task_id=task_id,
+            pin=pin,
+            input=f"engine-queue/inputs/{self.run}/{pin_id}/source.tar",
+            image=self.args.image,
+            deadline=self.args.deadline,
+            question_deadline=self.args.question_deadline,
+        )
+        try:
+            self.client.put(task["input"], pack(checkout), {})
+            put(self.client, PENDING + task_id, task)
+            end = self.clock() + self.args.queue_timeout
+            while self.clock() < end:
+                result = read(self.client, result_key(task))
+                if result is not None:
+                    (output / "result.json").write_text(json.dumps(result))
+                    self.client.delete(task["input"])
+                    return subprocess.CompletedProcess([], 0, "", ""), result.get("worker")
+                self.pause(2)
+            # Removing pending cancels work that has not been claimed; retain source for a live lease.
+            self.client.delete(PENDING + task_id)
+            return None, None
+        except Exception as exc:
+            # SDK errors can contain endpoint URLs or credentials. Do not persist their text.
+            raise OSError("queue transport failed") from exc
+
+    def status(self):
+        return status(self.client)

benchmarks/learn/engine_worker_service.py

@@ -0,0 +1,315 @@
+"""Image-resident queue workers. Requires strongly consistent S3 GET/PUT/LIST.
+
+Per-task Lamport bakery tickets use only ordinary object operations. A choosing
+marker precedes ticket selection; contenders wait for choosing peers and lower
+(ticket, worker) pairs. Late arrivals cannot displace a running owner. Leases
+assume bounded clock skew (less than MARGIN / 2) and a worker killed by its deadline.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import ctypes
+import io
+import json
+import logging
+import multiprocessing
+import os
+import signal
+import socket
+import tarfile
+import tempfile
+import threading
+import time
+import uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+from engine_trace_worker import unit_record, worker
+
+from openultrasast.plane.memory import S3Client, s3_settings
+
+PENDING = "engine-queue/pending/"
+CLAIMED = "engine-queue/claimed/"
+RESULTS = "engine-results/"
+MARGIN = 60
+LOG = logging.getLogger(__name__)
+
+
+def client_from_env():
+    settings = s3_settings(os.environ)
+    bucket = os.environ.get("S3_BUCKET")
+    if not bucket:
+        raise ValueError("S3_BUCKET is required")
+    return S3Client(bucket=bucket, **settings)
+
+
+def read(client, key):
+    value = client.get(key)
+    return json.loads(value[0]) if value else None
+
+
+def put(client, key, value):
+    client.put(key, json.dumps(value).encode(), {})
+
+
+def result_key(task):
+    return f"{RESULTS}{task['run']}/{task['id']}.json"
+
+
+def pack(root):
+    stream = io.BytesIO()
+    with tarfile.open(fileobj=stream, mode="w") as archive:
+        archive.add(root, arcname=".")
+    return stream.getvalue()
+
+
+def unpack(data, root):
+    with tarfile.open(fileobj=io.BytesIO(data)) as archive:
+        # Source snapshots need only regular files and directories, never links/devices.
+        for member in archive.getmembers():
+            target = (root / member.name).resolve()
+            if not target.is_relative_to(root.resolve()) or not (member.isfile() or member.isdir()):
+                raise ValueError("unsafe source archive")
+        archive.extractall(root, filter="data")
+
+
+class Queue:
+    def __init__(self, client, identity=None, *, now=time.time, pause=time.sleep, stop=None):
+        self.client = client
+        self.identity = identity or f"{socket.gethostname()}-{uuid.uuid4().hex}"
+        self.now, self.pause = now, pause
+        self.stop = stop or threading.Event()
+
+    def markers(self, task_id=None):
+        result = []
+        for key in self.client.keys(CLAIMED):
+            if task_id is None or key.rsplit("/", 1)[-1] == task_id:
+                marker = read(self.client, key)
+                if marker:
+                    result.append((key, marker))
+        return result
+
+    def tasks(self):
+        tasks = {}
+        for key in self.client.keys(PENDING):
+            task = read(self.client, key)
+            if task:
+                tasks[task["task_id"]] = task
+        # A dead worker may have removed pending already. Its task remains in the lease.
+        for _, marker in self.markers():
+            if marker["expires"] < self.now():
+                task = marker["task"]
+                tasks.setdefault(task["task_id"], task)
+        return list(tasks.values())
+
+    def claim(self, task):
+        if read(self.client, result_key(task)) is not None or self.stop.is_set():
+            return None
+        key = f"{CLAIMED}{self.identity}/{task['task_id']}"
+        marker = dict(task=task, ticket=0, expires=self.now() + task["deadline"] + MARGIN)
+        put(self.client, key, marker)
+        try:
+            live = [m for _, m in self.markers(task["task_id"]) if m["expires"] > self.now()]
+            marker["ticket"] = 1 + max((m["ticket"] for m in live), default=0)
+            put(self.client, key, marker)
+            while not self.stop.is_set():
+                # Do not start once the execution budget no longer fits within our lease.
+                if self.now() >= marker["expires"] - task["deadline"] - MARGIN / 2:
+                    break
+                peers = [(k, m) for k, m in self.markers(task["task_id"]) if k != key and m["expires"] > self.now()]
+                if any(m["ticket"] == 0 for _, m in peers):
+                    self.pause(0.1)
+                    continue
+                if any((m["ticket"], k) < (marker["ticket"], key) for k, m in peers):
+                    break  # loser backs off and can try another pending task
+                if read(self.client, key) != marker or read(self.client, result_key(task)) is not None:
+                    break
+                self.client.delete(PENDING + task["task_id"])
+                return key
+        except Exception:
+            self.client.delete(key)
+            raise
+        self.client.delete(key)
+        return None
+
+    def process_one(self, execute=None):
+        for task in self.tasks():
+            key = self.claim(task)
+            if key is None:
+                continue
+            try:
+                lease = read(self.client, key)
+                if lease is None:
+                    continue
+                budget = min(task["deadline"], lease["expires"] - self.now() - MARGIN / 2)
+                if budget <= 0:
+                    continue  # Leave the expired lease's task available to recovery.
+                result = (execute or execute_task)(self.client, {**task, "runtime_budget": budget})
+                result.update(executor="queue", worker=self.identity, pod=socket.gethostname(), image=os.environ.get("ENGINE_IMAGE", ""))
+                put(self.client, result_key(task), result)
+                # Publish the result before removing either the input or the claim.
+                self.client.delete(task["input"])
+                for old_key, old in self.markers(task["task_id"]):
+                    if old_key == key or old["expires"] < self.now():
+                        self.client.delete(old_key)
+                return True
+            except Exception:
+                # Keep the lease and task for recovery, without logging credential-bearing errors.
+                LOG.error("task failed; lease retained for recovery")
+                return False
+        return False
+
+
+def _child(pin, root, out, deadline, question_deadline):
+    os.setsid()
+    signal.signal(signal.SIGTERM, signal.SIG_DFL)
+    worker(pin, root, out, deadline, question_deadline)
+
+
+def adopt_orphans():
+    # Linux PR_SET_CHILD_SUBREAPER: escaped Joern sessions reparent here, not
+    # outside the service. There is exactly one task process tree per worker.
+    if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
+        raise OSError("cannot enable task descendant cleanup")
+
+
+def kill_children():
+    """Freeze parents before walking children, including new-session JVMs."""
+    seen = set()
+
+    def freeze(parent):
+        for entry in Path("/proc").iterdir():
+            if not entry.name.isdigit() or int(entry.name) in seen:
+                continue
+            try:
+                # comm can contain spaces or parentheses. Fields after it begin at state.
+                fields = (entry / "stat").read_text().rsplit(")", 1)[1].split()
+                if int(fields[1]) != parent:
+                    continue
+                pid = int(entry.name)
+                os.kill(pid, signal.SIGSTOP)
+                seen.add(pid)
+                freeze(pid)
+            except (FileNotFoundError, ProcessLookupError, PermissionError):
+                continue
+
+    # A child that exited during discovery can reparent descendants to this
+    # subreaper. Repeat until every remaining descendant is frozen.
+    while True:
+        count = len(seen)
+        freeze(os.getpid())
+        if len(seen) == count:
+            break
+    for pid in seen:
+        with contextlib.suppress(ProcessLookupError):
+            os.kill(pid, signal.SIGKILL)
+
+
+def execute_task(client, task):
+    adopt_orphans()
+    pin = task["pin"]
+    started = time.monotonic()
+    budget = task.get("runtime_budget", task["deadline"])
+    status, reason, code = "failed", "worker produced no result", None
+    with tempfile.TemporaryDirectory(prefix="engine-task-") as scratch:
+        root, out = Path(scratch) / "case", Path(scratch) / "result.json"
+        root.mkdir()
+        try:
+            if task["deadline"] > 900 or task["deadline"] <= 0:
+                raise ValueError("deadline exceeds deployment grace")
+            if task.get("image") != os.environ.get("ENGINE_IMAGE"):
+                raise ValueError("task image differs from deployed image")
+            source = client.get(task["input"])
+            if source is None:
+                raise ValueError("missing source")
+            unpack(source[0], root)
+            # The shipped worker also checks frontend input and graph output for every language.
+            files = [p for p in root.rglob("*") if p.is_file()]
+            if not files:
+                raise ValueError("empty source")
+            LOG.info("input proof: first file bytes=%d", len(files[0].read_bytes()))
+            if time.monotonic() - started >= budget:
+                status, reason = "timeout", "task input deadline"
+                raise TimeoutError(reason)
+            process = multiprocessing.get_context("fork").Process(target=_child, args=(pin, root, out, budget, task["question_deadline"]))
+            process.start()
+            try:
+                process.join(max(0, budget - (time.monotonic() - started)))
+                if process.is_alive():
+                    status, reason = "timeout", "task deadline"
+                else:
+                    reason = f"worker exit {process.exitcode}"
+                code = process.exitcode
+            finally:
+                # Joern creates separate sessions. Kill the whole descendant tree,
+                # not merely the Python wrapper's process group.
+                kill_children()
+                process.join()
+                # Reap adopted JVMs and their launchers after the direct child.
+                while True:
+                    try:
+                        os.waitpid(-1, 0)
+                    except ChildProcessError:
+                        break
+            record = json.loads(out.read_text()) if out.exists() else {**pin, "units": []}
+        except Exception:
+            record = {**pin, "units": []}
+            reason = "worker input or execution failed"
+        if not record.get("done") or code != 0:
+            previous = {u["unit"]: u for u in record["units"]}
+            record["units"] = [
+                previous[u["unit"]]
+                if code is None and previous.get(u["unit"], {}).get("status") in {"path", "asked-nothing"}
+                else unit_record(u, status if u["supported"] else "unsupported", reason if u["supported"] else u["reason"])
+                for u in pin["units"]
+            ]
+        record.update(done=True, container_exit=code, seconds=time.monotonic() - started)
+        return record
+
+
+def status(client):
+    return {
+        "pending": len(client.keys(PENDING)),
+        "claimed": len(client.keys(CLAIMED)),
+        "done": len([k for k in client.keys(RESULTS) if k.endswith(".json")]),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--port", type=int, default=8080)
+    args = parser.parse_args()
+    logging.basicConfig(level=logging.INFO)
+    stop = threading.Event()
+    signal.signal(signal.SIGTERM, lambda *_: stop.set())
+    signal.signal(signal.SIGINT, lambda *_: stop.set())
+    client = client_from_env()
+    queue = Queue(client, stop=stop)
+
+    class Health(BaseHTTPRequestHandler):
+        def do_GET(self):
+            self.send_response(200 if self.path == "/healthz" and not stop.is_set() else 503)
+            self.end_headers()
+
+        def log_message(self, *_):
+            pass
+
+    server = ThreadingHTTPServer(("0.0.0.0", args.port), Health)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    try:
+        while not stop.is_set():
+            try:
+                if not queue.process_one():
+                    stop.wait(2)
+            except Exception:
+                LOG.error("queue unavailable; retrying")
+                stop.wait(5)
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/learn/joern_leak_audit.py

@@ -135,9 +135,7 @@ def audit(units, results, *, expected_rows=501, expected_pairs=195):
         counts["stale"] += stale
         counts["unfinished"] += unfinished
         counts["skipped"] += stale or unfinished
-        files.append(
+        files.append((path.name, len(raw)))
-            {"path": str(path), "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest(), "stale": stale, "unfinished": unfinished}
-        )
         if stale or unfinished:
             continue
         counts["accepted_files"] += 1
@@ -156,8 +154,6 @@ def audit(units, results, *, expected_rows=501, expected_pairs=195):
                 **unit,
                 "source": row["source"],
                 "status": row["status"],
-                "repo": record["repo"],
-                "pin": record["pin"],
                 "pin_role": row.get("pin_role"),
                 **features(row),
             }
@@ -181,8 +177,9 @@ def audit(units, results, *, expected_rows=501, expected_pairs=195):
         "inputs": {
             **{k: counts[k] for k in ("accepted_files", "result_rows_read", "stale", "unfinished", "skipped")},
             "files_read": len(files),
-            "bytes_read": sum(f["bytes"] for f in files),
+            "bytes_read": sum(size for _, size in files),
-            "files": files,
+            # Hash a canonical inventory; filenames and paths may identify repositories.
+            "files_sha256": hashlib.sha256(json.dumps(sorted(files), separators=(",", ":")).encode()).hexdigest(),
         },
         "semantics": {
             "signals": "Shortest nonempty trace (first on ties); its sanitizer, guard and corresponding witness sink call target.",
@@ -195,7 +192,7 @@ def audit(units, results, *, expected_rows=501, expected_pairs=195):
             "limits": "Exploratory associations, not proof of leakage; running input snapshot, no multiple-testing correction.",
         },
         **scopes,
-        "units": rows,
+        "units": [{k: r[k] for k in ("unit", "pair", "family", "label", "fold", "source", "status", *SIGNALS, "sink_kind")} for r in rows],
     }
 
 
@@ -208,7 +205,7 @@ def main():
     raw = args.units.read_bytes()
     units = [json.loads(line) for line in raw.splitlines() if line.strip()]
     report = audit(units, args.results)
-    report["units_input"] = {"path": str(args.units), "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
+    report["units_input"] = {"bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest()}
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
     overall, inputs = report["overall"], report["inputs"]
@@ -230,7 +227,7 @@ def main():
             return brief_metric(value)
 
         lines.append(f"{signal}: all={brief(overall['signals'][signal])}; both asked={brief(overall['both_asked_signals'][signal])}")
-    lines.append(f"Family/source metrics and per-file read evidence: {args.output}")
+    lines.append(f"Family/source metrics and input inventory digest: {args.output}")
     print("\n".join(lines))
 
 

benchmarks/learn/slice_leak_audit.py

@@ -141,7 +141,9 @@ def audit(store: MemoryStore, units: Sequence[Mapping[str, Any]], *, expected_ro
         "limitations": (
             "Flat IR: no dominance/guards, textual branch order, potential parameter inputs, conservative call returns; no alias analysis."
         ),
-        "units": rows,
+        "units": [
+            {k: r[k] for k in ("unit", "pair", "family", "label", "fold", "source", "status", "has_flow", "slice_length")} for r in rows
+        ],
     }
 
 

docs/rustfs.md

@@ -214,3 +214,42 @@ bucket `OUSAST_MEMORY_TEST_BUCKET`, else `S3_BUCKET`. The tests:
 - never configure the bucket.
 
 More on the host setup: [ax on this host](ops/ax/README.md).
+
+## Scoped account for engine workers
+
+Create a **new account** for the k3s engine Deployment. Attach only this policy.
+Do not give it the existing memory-store account's policies.
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "ListEnginePrefixes",
+      "Effect": "Allow",
+      "Action": "s3:ListBucket",
+      "Resource": "arn:aws:s3:::sast-memory",
+      "Condition": {
+        "StringLike": {
+          "s3:prefix": ["engine-queue/", "engine-queue/*", "engine-results/", "engine-results/*"]
+        }
+      }
+    },
+    {
+      "Sid": "EngineObjectsOnly",
+      "Effect": "Allow",
+      "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
+      "Resource": [
+        "arn:aws:s3:::sast-memory/engine-queue/*",
+        "arn:aws:s3:::sast-memory/engine-results/*"
+      ]
+    }
+  ]
+}
+```
+
+The cluster can read and write only queued engine inputs, claims, and results.
+It cannot list or read the rest of the memory store. It cannot change bucket
+settings. The host queue dispatcher can use the same scoped account. Store its
+keys in the `ousast-engine-s3` Secret; no key values belong in manifests.
+Deployment and coordinator commands are in `ops/k8s/engine/README.md`.

ops/k8s/engine/README.md

@@ -0,0 +1,85 @@
+# Engine workers on k3s
+
+Build the new **Engine image** workflow on main (or dispatch it), then copy its
+`ghcr.io/<owner>/openultrasast@sha256:...` job-summary value. The old September
+image does not include boto3 or this service. The release workflow still builds
+version tags. The Dockerfile copies the worker and parser under `/app/benchmarks/learn`
+and installs `.[semantic,s3]`; no analyzer tarball is sent by the dispatcher.
+
+Use two schedulable nodes with at least 3 GiB available each. Required hostname
+anti-affinity places one worker on each node. Updates use Recreate so a third
+worker does not wait forever for a third node. Local scratch uses the container's
+filesystem, with no host mount or Kubernetes API access.
+
+Create a **new scoped account** using the policy in `docs/rustfs.md`. Export its
+`S3_ENDPOINT`, `S3_BUCKET=sast-memory`, `AWS_ACCESS_KEY_ID`, and
+`AWS_SECRET_ACCESS_KEY` in the coordinator shell. Do not use the memory-store
+account. The following commands are for the coordinator; they were not run by the
+implementation agent.
+
+```sh
+export ENGINE_IMAGE='ghcr.io/<owner>/openultrasast@sha256:<digest-from-workflow>'
+# From the repository root, edit the digest parameter without changing the template:
+export ENGINE_OVERLAY="$(mktemp -d /tmp/ousast-engine.XXXXXX)"
+cp ops/k8s/engine/{namespace,deployment,kustomization}.yaml "$ENGINE_OVERLAY/"
+python - <<'PY'
+import os
+from pathlib import Path
+image, digest = os.environ['ENGINE_IMAGE'].split('@', 1)
+p = Path(os.environ['ENGINE_OVERLAY']) / 'kustomization.yaml'
+p.write_text(p.read_text().replace('ghcr.io/OWNER/openultrasast', image)
+             .replace('sha256:REPLACE_WITH_ENGINE_IMAGE_DIGEST', digest))
+PY
+kubectl apply -f ops/k8s/engine/namespace.yaml
+kubectl -n ousast-engine create secret generic ousast-engine-s3 \
+  --from-literal=S3_ENDPOINT="$S3_ENDPOINT" \
+  --from-literal=S3_BUCKET="$S3_BUCKET" \
+  --from-literal=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
+  --from-literal=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"
+kubectl apply -k "$ENGINE_OVERLAY"
+kubectl -n ousast-engine rollout status deployment/ousast-engine-worker
+kubectl -n ousast-engine get pods -o wide
+
+export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH
+export PYTHONPATH="$PWD/src"
+python benchmarks/learn/engine_trace.py --queue-status
+python benchmarks/learn/engine_trace.py --executor queue \
+  --image "$ENGINE_IMAGE" --parallel 2 --deadline 900 --queue-timeout 3600 \
+  --units plane/experiments/exp-005-graph-slice.units.jsonl \
+  --out "$HOME/ousast-results/plane/engine-trace-queue"
+```
+
+The dispatcher uses the existing local memory examples by default. Add
+`--examples /path/to/examples.jsonl` for a frozen example snapshot. Materialization
+uses locally cached source pins, just as the Docker executor does. `--executor
+k8s-jobs` preserves the old Job lane (`k8s` is its legacy alias); `queue` requires
+no kubectl access. Resume skips completed local units and retries failed or timed
+out units. `progress.json`, `STOP`, filtering, and `--summary` retain their existing
+meaning. Each invocation uses a new run ID, so an old remote result cannot satisfy
+a new retry. The dispatcher records worker and pod identity in each pin JSON.
+
+Tasks contain the pin, units, prepared questions, source object key, and deadlines.
+Results live at `engine-results/<run>/<pin-id>.json`; inputs are removed after
+publication. The Docker lane retains JSON and a log, not a result tar, so the queue
+also returns JSON. Queue timeout records timeout locally and removes pending;
+an already claimed task finishes remotely and cleans its input. Orphaned inputs
+from a failed submission can be removed under `engine-queue/inputs/` after checking
+that no pending task or claim references them.
+
+Claims use per-worker choosing markers followed by ordered tickets. A worker waits
+for choosing peers and every lower ticket before removing pending and starting.
+This requires strongly consistent GET/PUT/LIST from RustFS, including paginated
+LIST, and synchronized node clocks with skew below 30 seconds. Half of the 60-second lease margin is reserved for clock
+skew; the remaining execution budget is checked again after claiming.
+It does not require conditional writes, object tags, S3 Select, or bucket-admin
+permissions. A dead worker's task is recovered from its expired claim. Execution
+runs in a child process. Linux subreaping and a frozen descendant walk kill its
+entire tree, including Joern's separate sessions, at the task deadline before the
+lease expires. As with any unfenced lease, a node suspended beyond its lease must be
+terminated before it rejoins; the protocol cannot guarantee exclusivity against
+arbitrary process suspension or an inconsistent object store.
+
+SIGTERM stops new claims and lets the current task finish. The 900-second
+termination grace matches the maximum task deadline. Keep these values aligned
+if changing the deployment. `/healthz` serves both probes and becomes unready on
+shutdown. Errors omit SDK exception text to avoid exposing credentials.

ops/k8s/engine/deployment.yaml

@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ousast-engine-worker
+spec:
+  replicas: 2
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: ousast-engine-worker
+  template:
+    metadata:
+      labels:
+        app: ousast-engine-worker
+    spec:
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 900
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: ousast-engine-worker
+              topologyKey: kubernetes.io/hostname
+      containers:
+        - name: worker
+          image: engine-image
+          command: [python, /app/benchmarks/learn/engine_worker_service.py]
+          envFrom:
+            - secretRef:
+                name: ousast-engine-s3
+          env:
+            - name: ENGINE_IMAGE
+              value: engine-image
+          resources:
+            requests:
+              memory: 3Gi
+              cpu: '1'
+            limits:
+              memory: 3Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+          ports:
+            - name: health
+              containerPort: 8080
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: health
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: health
+            initialDelaySeconds: 15
+            periodSeconds: 10

ops/k8s/engine/kustomization.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: ousast-engine
+resources:
+  - namespace.yaml
+  - deployment.yaml
+# Deliberately non-pullable until the coordinator supplies the workflow's digest.
+images:
+  - name: engine-image
+    newName: ghcr.io/OWNER/openultrasast
+    digest: sha256:REPLACE_WITH_ENGINE_IMAGE_DIGEST
+replacements:
+  - source:
+      kind: Deployment
+      name: ousast-engine-worker
+      fieldPath: spec.template.spec.containers.0.image
+    targets:
+      - select:
+          kind: Deployment
+          name: ousast-engine-worker
+        fieldPaths:
+          - spec.template.spec.containers.0.env.0.value

ops/k8s/engine/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ousast-engine

ops/k8s/engine/secret.example.yaml

@@ -0,0 +1,9 @@
+# Reference only; excluded from kustomization. Create using the README command.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ousast-engine-s3
+  namespace: ousast-engine
+type: Opaque
+stringData: {}
+# Required keys: S3_ENDPOINT, S3_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY

tests/test_engine_queue.py

@@ -0,0 +1,290 @@
+"""Offline queue protocol, image imports and deployment contract controls."""
+
+from __future__ import annotations
+
+import importlib
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+with pytest.MonkeyPatch.context() as patch:
+    patch.syspath_prepend(str(ROOT / "benchmarks/learn"))
+    service = importlib.import_module("engine_worker_service")
+    transport = importlib.import_module("engine_trace_queue")
+
+
+class FakeS3:
+    def __init__(self):
+        self.objects = {}
+        self.lock = threading.Lock()
+        self.choosing = None
+
+    def get(self, key):
+        with self.lock:
+            return (self.objects[key], None) if key in self.objects else None
+
+    def put(self, key, data, labels):
+        with self.lock:
+            self.objects[key] = data
+        if self.choosing and key.startswith(service.CLAIMED) and json.loads(data)["ticket"] == 0:
+            self.choosing.wait(timeout=5)
+
+    def delete(self, key):
+        with self.lock:
+            self.objects.pop(key, None)
+
+    def keys(self, prefix):
+        with self.lock:
+            return sorted(k for k in self.objects if k.startswith(prefix))
+
+
+def task():
+    return dict(
+        run="run",
+        id="pin",
+        task_id="run-pin",
+        input="engine-queue/inputs/run/pin/source.tar",
+        deadline=10,
+        question_deadline=2,
+        pin={"repo": "owner/repo", "pin": "commit", "units": []},
+    )
+
+
+def test_engine_worker_racing_claimants_run_exactly_once():
+    store = FakeS3()
+    item = task()
+    service.put(store, service.PENDING + item["task_id"], item)
+    store.choosing = threading.Barrier(2)
+    called = []
+
+    def execute(client, entry):
+        called.append(entry["task_id"])
+        time.sleep(0.05)
+        return {**entry["pin"], "done": True, "instrument": {"bytes": 11}}
+
+    with ThreadPoolExecutor(2) as pool:
+        futures = [pool.submit(service.Queue(store, identity).process_one, execute) for identity in ["worker-b", "worker-a"]]
+        results = [f.result(timeout=10) for f in futures]
+    assert sum(results) == 1
+    assert called == ["run-pin"]
+    result = service.read(store, service.result_key(item))
+    assert result["worker"] in {"worker-a", "worker-b"}
+    assert not store.keys(service.CLAIMED)
+    assert not store.keys(service.PENDING)
+
+
+def test_engine_worker_expired_lease_recovered_without_pending():
+    store = FakeS3()
+    item = task()
+    service.put(store, "engine-queue/claimed/dead/run-pin", dict(task=item, ticket=1, expires=99))
+    queue = service.Queue(store, "new", now=lambda: 100)
+    assert queue.process_one(lambda *_: {**item["pin"], "done": True})
+    assert service.read(store, service.result_key(item))["worker"] == "new"
+    assert not store.keys(service.CLAIMED)
+
+
+def test_engine_worker_late_arrival_cannot_displace_running_owner():
+    store = FakeS3()
+    item = task()
+    first = service.Queue(store, "z-first")
+    key = first.claim(item)
+    assert key
+    stop = threading.Event()
+    later = service.Queue(store, "a-later", stop=stop)
+    with ThreadPoolExecutor(1) as pool:
+        future = pool.submit(later.claim, item)
+        assert future.result(timeout=3) is None
+        assert service.read(store, key) is not None
+        service.put(store, service.result_key(item), {"done": True})
+        store.delete(key)
+
+
+def test_engine_queue_dispatch_roundtrip_and_timeout(tmp_path):
+    store = FakeS3()
+    checkout, output = tmp_path / "case", tmp_path / "out"
+    checkout.mkdir()
+    output.mkdir()
+    (checkout / "app.py").write_text("print(1)\n")
+    args = SimpleNamespace(deadline=10, question_deadline=2, queue_timeout=2, image="image@sha256:abc")
+    unit = {"unit": "u", "supported": True, "reason": "", "family": "injection"}
+    pin = {"repo": "owner/repo", "pin": "sha", "units": [unit], "questions": []}
+
+    def execute(client, item):
+        unpacked = tmp_path / "unpacked"
+        unpacked.mkdir()
+        service.unpack(client.get(item["input"])[0], unpacked)
+        assert (unpacked / "app.py").read_bytes() == b"print(1)\n"
+        return {
+            **item["pin"],
+            "done": True,
+            "container_exit": 0,
+            "units": [service.unit_record(unit, "asked-nothing")],
+            "instrument": {"bytes": 9, "heap_mb": 2560},
+        }
+
+    queue = service.Queue(store, "pod-1")
+    dispatcher = transport.Dispatcher(args, store, pause=lambda _: queue.process_one(execute))
+    completed, worker = dispatcher.execute(checkout, output, pin, "pin")
+    assert completed.returncode == 0
+    assert worker == "pod-1"
+    result = json.loads((output / "result.json").read_text())
+    assert result["executor"] == "queue"
+    assert result["units"][0]["status"] == "asked-nothing"
+    assert result["instrument"]["heap_mb"] == 2560
+    assert dispatcher.status() == {"pending": 0, "claimed": 0, "done": 1}
+    assert not store.keys("engine-queue/inputs/")
+    ticks = iter([0, 3])
+    dispatcher = transport.Dispatcher(args, store, clock=lambda: next(ticks))
+    assert dispatcher.execute(checkout, output, pin, "other") == (None, None)
+    assert not store.keys(service.PENDING)
+
+
+def test_engine_worker_imports_without_yaml():
+    code = "import sys; sys.modules['yaml'] = None; import engine_worker_service; import engine_trace_worker"
+    done = subprocess.run(
+        [sys.executable, "-c", code],
+        capture_output=True,
+        text=True,
+        env={**os.environ, "PYTHONPATH": f"{ROOT / 'src'}:{ROOT / 'benchmarks/learn'}"},
+    )
+    assert done.returncode == 0, done.stderr
+
+
+def test_engine_k8s_render_contract():
+    base = ROOT / "ops/k8s/engine"
+    config = yaml.safe_load((base / "kustomization.yaml").read_text())
+    # Render this base's resource inclusion, namespace, image and replacement transforms offline.
+    resources = [yaml.safe_load((base / name).read_text()) for name in config["resources"]]
+    deployment = next(r for r in resources if r["kind"] == "Deployment")
+    deployment["metadata"]["namespace"] = config["namespace"]
+    image = config["images"][0]
+    pod = deployment["spec"]["template"]["spec"]
+    container = pod["containers"][0]
+    assert container["image"] == image["name"]
+    container["image"] = image["newName"] + "@" + image["digest"]
+    replacement = config["replacements"][0]
+    assert replacement["source"]["fieldPath"] == "spec.template.spec.containers.0.image"
+    assert replacement["targets"][0]["fieldPaths"] == ["spec.template.spec.containers.0.env.0.value"]
+    container["env"][0]["value"] = container["image"]
+    assert "@sha256:" in container["image"]
+    assert deployment["spec"]["replicas"] == 2
+    assert container["resources"] == {"requests": {"memory": "3Gi", "cpu": "1"}, "limits": {"memory": "3Gi"}}
+    affinity = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"][0]
+    assert affinity["topologyKey"] == "kubernetes.io/hostname"
+    assert affinity["labelSelector"]["matchLabels"] == deployment["spec"]["template"]["metadata"]["labels"]
+    assert pod["terminationGracePeriodSeconds"] == 900
+    assert container["envFrom"] == [{"secretRef": {"name": "ousast-engine-s3"}}]
+    for probe in ["readinessProbe", "livenessProbe"]:
+        assert container[probe]["httpGet"] == {"path": "/healthz", "port": "health"}
+    assert not pod["automountServiceAccountToken"]
+    assert pod["securityContext"]["runAsNonRoot"]
+    assert "hostPath" not in json.dumps(resources)
+    assert "privileged" not in json.dumps(resources)
+    assert all(r["kind"] != "Secret" for r in resources)
+    secret = yaml.safe_load((base / "secret.example.yaml").read_text())
+    assert secret["stringData"] == {}
+    assert "AWS_SECRET_ACCESS_KEY" not in json.dumps(resources)
+
+
+def test_engine_image_and_rustfs_policy():
+    dockerfile = (ROOT / "Dockerfile").read_text()
+    assert "COPY benchmarks ./benchmarks" in dockerfile
+    assert ".[semantic,s3]" in dockerfile
+    for name in ["engine_trace_worker.py", "engine_trace_parse.py", "engine_worker_service.py"]:
+        assert (ROOT / "benchmarks/learn" / name).stat().st_size > 0
+    doc = (ROOT / "docs/rustfs.md").read_text().split("## Scoped account for engine workers")[1]
+    policy = json.loads(doc.split("```json\n")[1].split("```")[0])
+    listing, objects = policy["Statement"]
+    assert listing["Action"] == "s3:ListBucket"
+    assert set(listing["Condition"]["StringLike"]["s3:prefix"]) == {
+        "engine-queue/",
+        "engine-queue/*",
+        "engine-results/",
+        "engine-results/*",
+    }
+    assert set(objects["Action"]) == {"s3:GetObject", "s3:PutObject", "s3:DeleteObject"}
+    assert set(objects["Resource"]) == {"arn:aws:s3:::sast-memory/engine-queue/*", "arn:aws:s3:::sast-memory/engine-results/*"}
+
+
+@pytest.mark.parametrize("hang", [False, True])
+def test_engine_worker_process_deadline_and_shipped_schema(tmp_path, monkeypatch, hang):
+    store = FakeS3()
+    item = task()
+    item["image"] = "image@sha256:abc"
+    item["deadline"] = 0.1 if hang else 5
+    item["pin"]["units"] = [{"unit": "u", "supported": True, "reason": ""}]
+    (tmp_path / "app.py").write_text("print(1)\n")
+    store.put(item["input"], service.pack(tmp_path), {})
+    monkeypatch.setenv("ENGINE_IMAGE", item["image"])
+
+    def fake_worker(pin, root, out, deadline, question_deadline):
+        assert (root / "app.py").read_bytes() == b"print(1)\n"
+        if hang:
+            time.sleep(10)
+        out.write_text(json.dumps({**pin, "done": True, "units": [service.unit_record(pin["units"][0], "asked-nothing")]}))
+
+    monkeypatch.setattr(service, "worker", fake_worker)
+    result = service.execute_task(store, item)
+    assert result["done"]
+    assert result["units"][0]["status"] == ("timeout" if hang else "asked-nothing")
+    assert result["container_exit"] == (None if hang else 0)
+    assert result["seconds"] < 5
+
+
+def test_engine_worker_stopping_does_not_claim():
+    store = FakeS3()
+    item = task()
+    service.put(store, service.PENDING + item["task_id"], item)
+    stop = threading.Event()
+    stop.set()
+    assert not service.Queue(store, stop=stop).process_one(lambda *_: pytest.fail("claimed after SIGTERM"))
+    assert store.keys(service.PENDING)
+    assert not store.keys(service.CLAIMED)
+
+
+def test_engine_worker_rejects_archive_links_and_escape(tmp_path):
+    import io
+    import tarfile
+
+    for name, kind in [("../escape", tarfile.REGTYPE), ("link", tarfile.SYMTYPE)]:
+        stream = io.BytesIO()
+        with tarfile.open(fileobj=stream, mode="w") as archive:
+            member = tarfile.TarInfo(name)
+            member.type = kind
+            member.linkname = "/etc/passwd"
+            archive.addfile(member)
+        with pytest.raises(ValueError, match="unsafe source archive"):
+            service.unpack(stream.getvalue(), tmp_path)
+
+
+def test_engine_worker_timeout_kills_separate_session_descendant(tmp_path, monkeypatch):
+    store = FakeS3()
+    item = task()
+    item.update(image="image@sha256:abc", deadline=0.5)
+    item["pin"]["units"] = [{"unit": "u", "supported": True, "reason": ""}]
+    (tmp_path / "app.py").write_text("print(1)\n")
+    store.put(item["input"], service.pack(tmp_path), {})
+    monkeypatch.setenv("ENGINE_IMAGE", item["image"])
+    pid_file = tmp_path / "escaped-pid"
+
+    def fake_worker(*_):
+        child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"], start_new_session=True)
+        pid_file.write_text(str(child.pid))
+        time.sleep(30)
+
+    monkeypatch.setattr(service, "worker", fake_worker)
+    result = service.execute_task(store, item)
+    assert result["units"][0]["status"] == "timeout"
+    assert pid_file.exists(), "instrument: fake worker never launched its descendant"
+    pid = int(pid_file.read_text())
+    assert not Path(f"/proc/{pid}").exists(), "escaped session survived the task deadline"

tests/test_engine_trace.py

@@ -534,7 +534,15 @@ def test_trace_real_unsupported_run_then_typescript_opt_in_replans(tmp_path, mon
     first = json.loads(result_file.read_text())
     assert first["done"] and first["units"][0]["status"] == "unsupported"
     progress = json.loads((out / "progress.json").read_text())
-    assert progress == {"pins_done": 1, "pins_total": 1, "last_pin": {"repo": u["repo"], "pin": u["pin"]}, "statuses": {"unsupported": 1}}
+    assert progress == {
+        "pins_done": 1,
+        "pins_total": 1,
+        "last_pin": {"repo": u["repo"], "pin": u["pin"]},
+        "statuses": {"unsupported": 1},
+        "lanes": {"docker": 1},
+        "pins_in_flight": 0,
+        "active_lanes": {},
+    }
     assert trace.make_plan(trace.join_units([u], [example]), out) == []
 
     called = []
@@ -1161,3 +1169,286 @@ def test_overlay_engine_bug_allows_one_taint_only_attempt(tmp_path, monkeypatch)
     monkeypatch.setattr(JoernBackend, "_batch_once", batch)
     assert backend._batch_once(tmp_path / "cpg.bin", "taint", {"a": {}}) == {"a": []}
     assert len(backend.retries) == 1
+
+
+def k8s_args(tmp_path, **overrides):
+    from types import SimpleNamespace
+
+    return SimpleNamespace(
+        **{
+            "executor": "k8s",
+            "parallel": 2,
+            "kube_context": "test-context",
+            "kube_namespace": "ousast-engine",
+            "image": "ghcr.io/norandom/openultrasast:2.0.1",
+            "deadline": 900,
+            "question_deadline": 120,
+            "keep_jobs": False,
+            "out": tmp_path,
+            **overrides,
+        }
+    )
+
+
+def test_kubernetes_manifests_keep_urls_in_secret(tmp_path):
+    from engine_trace_k8s import BOOTSTRAP, manifests
+
+    args = k8s_args(tmp_path)
+    urls = {name: "https://store/" + name + "?signature=secret" for name in ("SOURCE_URL", "ANALYZER_URL", "RESULT_URL")}
+    job, secret = manifests("job", "run", args, urls, {})
+    spec = job["spec"]
+    pod = spec["template"]["spec"]
+    container = pod["containers"][0]
+    assert spec["activeDeadlineSeconds"] == 900
+    assert spec["backoffLimit"] == 0 and spec["ttlSecondsAfterFinished"] > 0
+    assert pod["restartPolicy"] == "Never" and not pod["automountServiceAccountToken"]
+    assert pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"]
+    assert container["image"] == args.image
+    assert pod["securityContext"] == {"runAsUser": 1000, "runAsGroup": 1000, "fsGroup": 1000}
+    assert {m["mountPath"] for m in container["volumeMounts"]} == {"/case", "/frozen", "/out"}
+    assert all(v["emptyDir"] == {} for v in pod["volumes"])
+    assert container["resources"] == {"requests": {"memory": "3Gi", "cpu": "1"}, "limits": {"memory": "3Gi"}}
+    assert secret["stringData"] == urls
+    assert all(url not in json.dumps(job) for url in urls.values())
+    assert "boto3" not in BOOTSTRAP and "yaml" not in BOOTSTRAP
+    compile(BOOTSTRAP, "<bootstrap>", "exec")
+
+
+@pytest.mark.parametrize("failure", [False, True])
+def test_kubernetes_host_pin_success_failure_and_cleanup(tmp_path, monkeypatch, failure):
+    import io
+    import tarfile
+
+    from engine_trace_k8s import Kubernetes
+
+    from openultrasast.plane.memory import S3Store
+
+    objects, calls, expiries = {}, [], []
+
+    class Client:
+        def put(self, key, data, labels):
+            objects[key] = data
+
+        def get(self, key):
+            return (objects[key], None) if key in objects else None
+
+        def delete(self, key):
+            objects.pop(key, None)
+
+        def presign(self, method, key, expires):
+            expiries.append(expires.total_seconds())
+            return "https://store/" + key + "?secret"
+
+    store = object.__new__(S3Store)
+    store.client, store.prefix = Client(), "prefix"
+    args = k8s_args(tmp_path)
+    transport = Kubernetes(args, store)
+    args.kubernetes = transport
+    pin = trace.make_plan([unit()], tmp_path)[0]
+    source = tmp_path / "frozen"
+    source.mkdir()
+    (source / "worker.py").write_text("print(42)")
+
+    class Inputs:
+        def materialize(self, pin, target):
+            target.mkdir()
+            (target / "app.py").write_text("def run(): pass")
+            return {"bytes": 15, "files": 1}
+
+    def kubectl(command, **kwargs):
+        assert command[:5] == ["kubectl", "--context", "test-context", "--namespace", "ousast-engine"]
+        calls.append(command[5:])
+        action = command[5:]
+        stdout = "{}"
+        if action[0] == "create":
+            manifest = json.loads(kwargs["input"])
+            if manifest["kind"] == "Secret":
+                assert manifest["metadata"]["ownerReferences"][0]["uid"] == "job-uid"
+            if manifest["kind"] == "Job":
+                stdout = json.dumps({"metadata": {"uid": "job-uid"}})
+                env = manifest["spec"]["template"]["spec"]["containers"][0]["env"]
+                prepared = json.loads(next(e["value"] for e in env if e["name"] == "PIN"))
+                result = {**prepared, "done": True, "units": [worker.unit_record(u, "asked-nothing", "") for u in prepared["units"]]}
+                result_bytes = json.dumps(result).encode()
+                stream = io.BytesIO()
+                with tarfile.open(fileobj=stream, mode="w") as archive:
+                    entry = tarfile.TarInfo("result.json")
+                    entry.size = len(result_bytes)
+                    archive.addfile(entry, io.BytesIO(result_bytes))
+                source_key = next(key for key in objects if key.endswith("source.tar"))
+                with tarfile.open(fileobj=io.BytesIO(objects[source_key])) as archive:
+                    assert archive.extractfile("app.py").read() == b"def run(): pass"
+                objects[source_key.replace("source.tar", "result.tar")] = stream.getvalue()
+        if action[:2] == ["get", "job"]:
+            stdout = json.dumps(
+                {"status": {"conditions": [{"type": "Failed", "status": "True", "reason": "DeadlineExceeded"}]}}
+                if failure
+                else {"status": {"succeeded": 1}}
+            )
+        if action[:2] == ["get", "pods"]:
+            stdout = json.dumps({"items": [{"spec": {"nodeName": "worker-1"}}]})
+        if action[0] == "logs":
+            assert "--tail=40" in action
+            stdout = "last worker line https://store/private?secret"
+        return subprocess.CompletedProcess(command, 0, stdout, "")
+
+    monkeypatch.setattr(subprocess, "run", kubectl)
+    monkeypatch.setattr(trace, "prepare_questions", lambda *a: {"a": {}})
+    assert trace.run_pin(pin, source, {"head": "frozen"}, Inputs(), tmp_path, args) is failure
+    result = json.loads((tmp_path / trace.pin_name(pin)).read_text())
+    assert result["done"] and result["executor"] == "k8s" and result["node"] == "worker-1"
+    assert result["analyzer"] == {"head": "frozen"}
+    assert result["container_exit"] == int(failure)
+    assert result["units"][0]["status"] == ("failed" if failure else "asked-nothing")
+    if failure:
+        assert "DeadlineExceeded" in result["units"][0]["reason"]
+        assert "last worker line" in result["units"][0]["reason"]
+        assert "?secret" not in json.dumps(result)
+    else:
+        assert trace.make_plan([unit()], tmp_path) == []
+    assert not objects
+    assert any(c[:2] == ["delete", "secret"] for c in calls)
+    assert any(c[:2] == ["delete", "job"] for c in calls)
+    assert min(expiries) >= args.deadline + 900
+
+
+@pytest.mark.parametrize("executor,parallel", [("docker", 1), ("k8s", 2), ("mixed", 3)])
+def test_scheduler_bounds_lanes_and_progress(tmp_path, monkeypatch, executor, parallel):
+    import threading
+    import time
+    from collections import Counter
+
+    active, peak = Counter(), Counter()
+    lock = threading.Lock()
+    args = k8s_args(tmp_path, executor=executor, parallel=parallel)
+    plan = trace.make_plan([unit(str(i), str(i)) for i in range(9)], tmp_path)
+
+    def run(pin, source, provenance, inputs, out, options):
+        with lock:
+            active[options.executor] += 1
+            active["total"] += 1
+            for key, value in active.items():
+                peak[key] = max(peak[key], value)
+        time.sleep(0.02)
+        trace.save_pin(
+            out / trace.pin_name(pin), {**pin, "done": True, "units": [worker.unit_record(u, "asked-nothing", "") for u in pin["units"]]}
+        )
+        with lock:
+            active[options.executor] -= 1
+            active["total"] -= 1
+        return False
+
+    monkeypatch.setattr(trace, "run_pin", run)
+    assert not trace.run_plan(plan, None, {}, None, args)
+    assert peak["total"] == parallel
+    if executor == "mixed":
+        assert peak["docker"] == 1 and peak["k8s"] == parallel - 1
+    progress = json.loads((tmp_path / "progress.json").read_text())
+    assert progress["pins_done"] == 9 and progress["pins_in_flight"] == 0
+    assert sum(progress["lanes"].values()) == parallel
+    assert trace.make_plan([unit(str(i), str(i)) for i in range(9)], tmp_path) == []
+
+
+@pytest.mark.parametrize("keep", [False, True])
+def test_k8s_check_roundtrip_and_retention(tmp_path, monkeypatch, capsys, keep):
+    from engine_trace_k8s import Kubernetes
+
+    from openultrasast.plane.memory import S3Store
+
+    data, manifests_seen, commands = {}, [], []
+
+    class Client:
+        def put(self, key, body, labels):
+            data[key] = body
+
+        def get(self, key):
+            return (b"x" * 1024, None)
+
+        def delete(self, key):
+            data.pop(key, None)
+
+        def presign(self, method, key, expires):
+            return "https://private/" + key
+
+    store = object.__new__(S3Store)
+    store.client, store.prefix = Client(), ""
+    runner = Kubernetes(k8s_args(tmp_path, keep_jobs=keep), store)
+
+    def fake(command, **kwargs):
+        commands.append(command)
+        action = command[5:]
+        result = {}
+        if kwargs.get("input"):
+            manifest = json.loads(kwargs["input"])
+            manifests_seen.append(manifest)
+            result = {"metadata": {"uid": "check-uid"}}
+        if action[:2] == ["get", "nodes"]:
+            result = {"items": [{"metadata": {"name": "node"}, "status": {"allocatable": {"memory": "6000000Ki"}}}]}
+        if action[:2] == ["get", "pods"]:
+            result = {"items": [{"spec": {"nodeName": "node"}}]}
+        if action[:2] == ["get", "job"]:
+            result = {"status": {"succeeded": 1}}
+        return subprocess.CompletedProcess(command, 0, json.dumps(result), "")
+
+    monkeypatch.setattr(subprocess, "run", fake)
+    assert runner.check() == 0
+    text = capsys.readouterr().out
+    assert "6000000Ki" in text and "1024" in text and "passed" in text
+    assert "https://" not in text
+    assert any(m["kind"] == "Namespace" for m in manifests_seen)
+    job = next(m for m in manifests_seen if m["kind"] == "Job")
+    assert {"name": "CHECK", "value": "1"} in job["spec"]["template"]["spec"]["containers"][0]["env"]
+    assert bool(data) is keep
+    assert any("delete" in c for c in commands) is not keep
+
+
+@pytest.mark.parametrize("outcome", ["success", "timeout", "failure"])
+def test_engine_queue_pin_persistence_resume_and_progress(tmp_path, monkeypatch, outcome):
+    from engine_trace_queue import Dispatcher
+    from engine_worker_service import Queue
+    from test_engine_queue import FakeS3
+
+    store = FakeS3()
+    args = k8s_args(tmp_path, executor="queue", queue_timeout=2, image="image@sha256:abc")
+    plan = trace.make_plan([unit()], tmp_path)
+
+    class Inputs:
+        def materialize(self, pin, target):
+            target.mkdir()
+            (target / "app.py").write_text("def run(): pass")
+            return {"files": 1, "bytes": 15}
+
+    def execute(client, task):
+        return {
+            **task["pin"],
+            "done": True,
+            "container_exit": 0,
+            "units": [worker.unit_record(u, "asked-nothing") for u in task["pin"]["units"]],
+        }
+
+    queue = Queue(store, "pod-worker")
+    if outcome == "timeout":
+        ticks = iter([0, 3])
+        args.dispatcher = Dispatcher(args, store, clock=lambda: next(ticks))
+    else:
+        args.dispatcher = Dispatcher(args, store, pause=lambda _: queue.process_one(execute))
+    if outcome == "failure":
+
+        def fail(*_):
+            raise RuntimeError("credential-must-not-appear")
+
+        monkeypatch.setattr(store, "put", fail)
+    monkeypatch.setattr(trace, "prepare_questions", lambda *_: {"a": {}})
+    monkeypatch.setattr(subprocess, "run", lambda *_a, **_k: pytest.fail("queue must not launch Docker/kubectl"))
+    assert trace.run_plan(plan, None, {"image": args.image}, Inputs(), args) is (outcome != "success")
+    result = json.loads((tmp_path / trace.pin_name(plan[0])).read_text())
+    assert result["executor"] == "queue"
+    assert result["units"][0]["status"] == {"success": "asked-nothing", "timeout": "timeout", "failure": "failed"}[outcome]
+    assert "credential-must-not-appear" not in json.dumps(result)
+    if outcome == "success":
+        assert result["worker"] == "pod-worker"
+        assert trace.make_plan([unit()], tmp_path) == []
+    else:
+        assert trace.make_plan([unit()], tmp_path, rerun_status=("failed", "timeout"))
+    progress = json.loads((tmp_path / "progress.json").read_text())
+    assert progress["pins_done"] == 1 and progress["pins_in_flight"] == 0

tests/test_joern_leak_audit.py

@@ -1,5 +1,6 @@
 """Synthetic, offline result snapshots; no analyzer, model or corpus access."""
 
+import hashlib
 import importlib
 import json
 from pathlib import Path
@@ -159,3 +160,31 @@ def test_bad_inputs_fail_loudly(audit_module, tmp_path):
     write_result(tmp_path, audit_module, [{**result(population[0], "path"), "label": 0}])
     with pytest.raises(ValueError, match="metadata mismatch"):
         run(audit_module, tmp_path, population)
+
+
+@pytest.mark.parametrize("done", [False, True])
+def test_record_uses_hashes_without_repository_identity(audit_module, tmp_path, done):
+    repo = "https://github.com/owner/name"
+    population = [{**u, "group": "owner/name", "repo": repo, "source": "population-v1"} for u in units(1)]
+    root = tmp_path / "github.com" / "owner" / "name"
+    root.mkdir(parents=True)
+    paths = []
+    for i, u in enumerate(population):
+        row = {**result(u, "path"), "source": "population-v1"}
+        path = write_result(root, audit_module, [row], f"name-{1 - i}", done=done)
+        record = json.loads(path.read_text())
+        record.update(repo=repo, pin="owner/name")
+        path.write_text(json.dumps(record))
+        paths.append(path)
+    report = run(audit_module, root, population)
+    serialized = json.dumps(report)
+    assert repo not in serialized and "owner/name" not in serialized
+    assert "files" not in report["inputs"]
+    inventory = sorted((p.name, p.stat().st_size) for p in paths)
+    assert report["inputs"]["files_sha256"] == hashlib.sha256(json.dumps(inventory, separators=(",", ":")).encode()).hexdigest()
+    assert report["inputs"]["files_read"] == 2
+    assert report["inputs"]["bytes_read"] == sum(size for _, size in inventory)
+    assert report["inputs"]["result_rows_read"] == (2 if done else 0)
+    assert report["by_source"]["population-v1"]["rows"] == 2
+    assert [r["unit"] for r in report["units"]] == [u["unit"] for u in population]
+    assert [r["pair"] for r in report["units"]] == [u["pair"] for u in population]

tests/test_learn_slice.py

@@ -150,7 +150,9 @@ def test_learn_slice_audit_reads_and_never_writes(audit_module: ModuleType, monk
 
     raw = b"   10  def f(x):\n   11      execute(x)\n"
     digest = hashlib.sha256(raw).hexdigest()
-    example = replace(corpus(groups=1)[0], excerpt_sha=digest, source="pairs", language="python", family="injection")
+    example = replace(
+        corpus(groups=1)[0], excerpt_sha=digest, source="pairs", language="python", family="injection", group="github.com/owner/name"
+    )
     monkeypatch.setattr(audit_module, "load_examples", lambda store: [example])
 
     class ReadOnly:
@@ -161,6 +163,8 @@ def test_learn_slice_audit_reads_and_never_writes(audit_module: ModuleType, monk
     units = [{"family": example.family, "label": example.label, "group": example.group, "unit": example.id, "pair": None, "fold": "f"}]
     report = audit_module.audit(ReadOnly(), units, expected_rows=1, expected_pairs=0)
     assert report["example_rows_read"] == report["matched_rows"] == report["excerpts_read"] == 1
+    assert "owner/name" not in json.dumps(report)
+    assert report["units"][0]["unit"] == example.id
     assert report["excerpt_bytes"] == len(raw)
     assert report["coverage"]["injection"]["flow"] == 1