feat(cpg): add a transaction-owned engine session and route it opt-in (M2 11.1, 11.3)

JVM startup, not analysis, is what a push scan spends its time on: each disposable
script invocation costs 13 to 23 seconds while the work takes under a second.

The session owns one Joern server per transaction on loopback with an OS-chosen
port and no published port. Three properties are load-bearing, and the probe that
settled the design demonstrated why each is needed.

An HTTP status is not an answer. The server returns success even for a compile
error, with the diagnostic in its body, so every request redirects the evaluated
code's console output into a fresh private file behind a nonce, and the nonce is
verified before the answer is accepted. The engine's body is retained for exactly
the case where the receipt is missing, without which that failure explains nothing.

A poisoned session answers nothing afterwards, and cancellation kills the process
group rather than the launcher, which this project has already run a machine out of
memory by neglecting.

What the probe established about the engine, after three iterations that each
failed loudly: runScript is absent from the server REPL; a shipped @main script
cannot be nested in a block expression, so definitions go to the top level
unwrapped; and the REPL shadows Console with io.joern.console.Console, which has no
withOut, so capture must qualify scala.Console. With that fixed, both mechanisms
answer identically to the disposable invocation, 0.8 s against 9.8 s.

The backend routes batch and single queries through the session only when
session_transport is enabled, which it is not by default. Any failure falls back to
the disposable invocation rather than reporting a zero, and each script is defined
under its own name because every script declares exec.

Equivalence for taint, dominance and configuration is task 11.2 and gates any
measured run. 1,391 tests passed, nine skipped. Rollout stays NO-GO.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

src/openultrasast/cpg/backend.py

@@ -36,6 +36,7 @@ from typing import Any, Protocol
 
 from ..model.contracts import ExecutionBudget
 from .artifact import GraphArtifact, GraphCensus, GraphIdentity
+from .session import EngineSession
 
 logger = logging.getLogger(__name__)
 
@@ -320,9 +321,15 @@ class JoernBackend:
     # Why the last build refused, so the driver can name it instead of reporting a bare `cpg_build_failed`.
     last_failure: str = ""
     execution_budget: ExecutionBudget | None = None
+    # M2 11.3: opt-in transport. Off by default, so shipped behaviour stays the disposable
+    # invocation this project has measured. A session that cannot answer falls back to it rather
+    # than reporting a zero, and what a session returns is the same payload or nothing at all.
+    session_transport: bool = False
     _diagnostics: list[str] = field(default_factory=list, init=False, repr=False)
     _scratch: list[Path] = field(default_factory=list, init=False, repr=False)
     _cancel_deadline: float | None = field(default=None, init=False, repr=False)
+    _session: EngineSession | None = field(default=None, init=False, repr=False)
+    _defined: set[str] = field(default_factory=set, init=False, repr=False)
 
     def available(self) -> bool:
         from .capability import has_cpg
@@ -1038,24 +1045,29 @@ class JoernBackend:
         except OSError as exc:
             logger.warning("could not stage cpg batch %s: %s", query, exc)
             return None
-        command = [
-            shutil.which("joern") or "joern",
-            self._heap_flag(),
-            "--script",
-            str(script),
-            "--param",
-            f"cpgFile={cpg_path}",
-            "--param",
-            f"requestsFile={requests_file}",
-        ]
-        for name, value in shared.items():
-            command += ["--param", f"{name}={value}"]
-        completed = self._run(command, timeout=self.query_timeout, cwd=cpg_path.parent)
-        if completed is None or completed.returncode != 0:
-            detail = (completed.stderr or "")[-400:] if completed is not None else "timeout"
-            logger.warning("cpg batch %s failed: %s", query, detail)
-            return None
-        parsed = extract_payload(completed.stdout or "")
+        # The session answers the same script with the same parameters, or nothing; nothing means
+        # the disposable invocation below runs exactly as it always has.
+        body = self._session_payload(cpg_path, query, {"cpgFile": str(cpg_path), "requestsFile": str(requests_file), **shared})
+        if body is None:
+            command = [
+                shutil.which("joern") or "joern",
+                self._heap_flag(),
+                "--script",
+                str(script),
+                "--param",
+                f"cpgFile={cpg_path}",
+                "--param",
+                f"requestsFile={requests_file}",
+            ]
+            for name, value in shared.items():
+                command += ["--param", f"{name}={value}"]
+            completed = self._run(command, timeout=self.query_timeout, cwd=cpg_path.parent)
+            if completed is None or completed.returncode != 0:
+                detail = (completed.stderr or "")[-400:] if completed is not None else "timeout"
+                logger.warning("cpg batch %s failed: %s", query, detail)
+                return None
+            body = completed.stdout or ""
+        parsed = extract_payload(body)
         if not self._time_available():
             return None
         if not isinstance(parsed, Mapping):
@@ -1075,19 +1087,71 @@ class JoernBackend:
         if not script.is_file():
             logger.warning("no such cpg query: %s", script)
             return None
-        command = [shutil.which("joern") or "joern", self._heap_flag(), "--script", str(script), "--param", f"cpgFile={cpg_path}"]
-        for key, value in sorted(params.items()):
-            command += ["--param", f"{key}={_render(value)}"]
-        # Joern writes a `workspace/` beside the working directory; run it inside the CPG's own scratch dir so
-        # it can never land in the repository being analysed.
-        completed = self._run(command, timeout=self.query_timeout, cwd=cpg_path.parent)
-        if completed is None or completed.returncode != 0:
-            detail = (completed.stderr or "")[-400:] if completed is not None else "timeout"
-            logger.warning("cpg query %s failed: %s", query, detail)
-            return None
-        parsed = extract_payload(completed.stdout or "")
+        body = self._session_payload(cpg_path, query, {"cpgFile": str(cpg_path), **params})
+        if body is None:
+            command = [shutil.which("joern") or "joern", self._heap_flag(), "--script", str(script), "--param", f"cpgFile={cpg_path}"]
+            for key, value in sorted(params.items()):
+                command += ["--param", f"{key}={_render(value)}"]
+            # Joern writes a `workspace/` beside the working directory; run it inside the CPG's own scratch dir so
+            # it can never land in the repository being analysed.
+            completed = self._run(command, timeout=self.query_timeout, cwd=cpg_path.parent)
+            if completed is None or completed.returncode != 0:
+                detail = (completed.stderr or "")[-400:] if completed is not None else "timeout"
+                logger.warning("cpg query %s failed: %s", query, detail)
+                return None
+            body = completed.stdout or ""
+        parsed = extract_payload(body)
         return parsed if self._time_available() else None
 
+    def _session_for(self, cpg_path: Path) -> EngineSession | None:
+        """The transaction's session, started on first use. None means use the disposable path."""
+        if not self.session_transport or self.execution_budget is None:
+            return None
+        if self._session is not None:
+            return self._session if self._session.alive else None
+        session = EngineSession(cpg_path.parent / "engine-session", self.execution_budget, self._jvm_env())
+        if not session.start():
+            self._note("session_unavailable:" + (session.failure or "unknown"))
+            session.close()
+            self._session = session
+            return None
+        self._session = session
+        return session
+
+    def _session_payload(self, cpg_path: Path, query: str, params: Mapping[str, object]) -> str | None:
+        """Run one shipped script in-session and return what it printed, or None to fall back.
+
+        The script is defined under a name of its own because every script declares `exec`, and a
+        second definition would shadow the first. Defining proves nothing: the receipt on the call
+        is the only proof, which is why any failure here simply falls back to the disposable path.
+        """
+        session = self._session_for(cpg_path)
+        if session is None:
+            return None
+        try:
+            source = (self.queries_dir / f"{query}.sc").read_text()
+        except OSError:
+            return None
+        symbol = f"ousast_{query}"
+        if query not in self._defined:
+            if session.define(source.replace("@main def exec(", f"def {symbol}(", 1), timeout=self.query_timeout) is None:
+                self._note("session_define_failed:" + query)
+                return None
+            self._defined.add(query)
+        arguments = ", ".join(f"{name} = {json.dumps(_render(value))}" for name, value in sorted(params.items()))
+        answer = session.evaluate(f"{symbol}({arguments})", timeout=self.query_timeout)
+        if answer is None:
+            self._note("session_query_failed:" + query + ":" + (session.failure or "unknown"))
+            return None
+        return answer.stdout
+
+    def close_session(self) -> None:
+        """End the transaction's session and reap its process group."""
+        if self._session is not None:
+            self._session.close()
+            self._session = None
+            self._defined.clear()
+
     def _note(self, reason: str) -> None:
         if reason not in self._diagnostics:
             self._diagnostics.append(reason)

src/openultrasast/cpg/session.py

@@ -0,0 +1,278 @@
+"""A transaction-owned Joern interpreter session on loopback (release milestone M2).
+
+Measured on this project: JVM startup, not analysis, is what a push scan spends its time on.
+Each disposable ``joern --script`` invocation costs 13 to 23 seconds while the work itself takes
+under a second, and one session starts in about 13 seconds and then answers in well under one.
+
+What this module does NOT do is decide anything. It is a transport. Every answer it returns is
+the same payload the disposable path would have produced, or nothing at all, and the caller keeps
+its existing fresh-build and disposable-query paths as the default and as the fallback.
+
+Three properties are load-bearing, and each exists because its absence has already produced a
+wrong answer somewhere in this project:
+
+* **An HTTP status is not an answer.** The lifecycle probe's first attempt received
+  ``success=true`` with an empty body because the payload went to the server's own log, and a
+  census of nothing would have been read as a census. So every request writes its output to a
+  fresh private file under a nonce, and the nonce is verified before the answer is accepted.
+* **A poisoned session answers nothing.** A session that crashed, timed out or returned an
+  unparseable body is marked failed, and every later request on it returns ``None`` rather than
+  an answer from an unknown state.
+* **The process group dies, not just the launcher.** ``joern`` is a shell script that starts a
+  JVM. Killing the script leaves the JVM holding its heap; this project has already run a machine
+  out of memory that way. Cancellation signals the group and waits inside the allowance.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import signal
+import socket
+import subprocess
+import time
+import urllib.error
+import urllib.request
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from openultrasast.model.contracts import ExecutionBudget
+
+logger = logging.getLogger(__name__)
+
+STARTUP_PROBE_SECONDS = 5.0
+STARTUP_POLL_SECONDS = 0.25
+
+
+def _free_port() -> int:
+    """Let the OS choose, so concurrent transactions cannot collide on a fixed port."""
+    with socket.socket() as probe:
+        probe.bind(("127.0.0.1", 0))
+        return int(probe.getsockname()[1])
+
+
+@dataclass
+class SessionAnswer:
+    """One verified answer: the stdout the evaluated code produced, and what it cost."""
+
+    stdout: str
+    seconds: float
+
+
+@dataclass
+class EngineSession:
+    """One Joern server owned by one transaction. Not reusable across transactions."""
+
+    scratch: Path
+    budget: ExecutionBudget
+    env: dict[str, str]
+    joern: str = "joern"
+    startup_timeout_seconds: float = 180.0
+    failure: str = ""
+    startup_seconds: float = 0.0
+    loaded_graph: str = ""
+    requests: int = 0
+    # What the engine printed for the most recent request. The server answers `success` even for a
+    # compile error and puts the diagnostic here, so discarding it leaves a missing receipt with no
+    # explanation and nothing to debug. Kept for exactly that reason.
+    last_body: str = ""
+    _process: subprocess.Popen[str] | None = field(default=None, init=False, repr=False)
+    _port: int = field(default=0, init=False, repr=False)
+    _log: Any = field(default=None, init=False, repr=False)
+
+    @property
+    def alive(self) -> bool:
+        return not self.failure and self._process is not None and self._process.poll() is None
+
+    def _poison(self, reason: str) -> None:
+        if not self.failure:
+            self.failure = reason
+            logger.warning("engine session poisoned: %s", reason)
+
+    def _remaining(self, limit: float) -> float:
+        return min(limit, self.budget.deadline_monotonic - time.monotonic())
+
+    def start(self) -> bool:
+        """Launch on loopback with an OS-chosen port. No published port, no network exposure."""
+        if self._process is not None:
+            return self.alive
+        if self._remaining(self.startup_timeout_seconds) <= 0:
+            self._poison("deadline_exhausted")
+            return False
+        self.scratch.mkdir(parents=True, exist_ok=True)
+        self._port = _free_port()
+        started = time.monotonic()
+        try:
+            self._log = (self.scratch / "session.log").open("w")
+            self._process = subprocess.Popen(  # noqa: S603 -- fixed argument array
+                [self.joern, "--server", "--server-host", "127.0.0.1", "--server-port", str(self._port)],
+                cwd=str(self.scratch),
+                env=self.env,
+                stdout=self._log,
+                stderr=subprocess.STDOUT,
+                text=True,
+                start_new_session=True,
+            )
+        except (OSError, subprocess.SubprocessError) as error:
+            self._poison("session_launch_failed:" + type(error).__name__)
+            return False
+        while True:
+            if self._process.poll() is not None:
+                self._poison("session_exited_during_startup")
+                return False
+            remaining = self._remaining(self.startup_timeout_seconds - (time.monotonic() - started))
+            if remaining <= 0:
+                self._poison("session_startup_timeout")
+                return False
+            if self._post("1", timeout=min(STARTUP_PROBE_SECONDS, remaining)) is not None:
+                self.startup_seconds = time.monotonic() - started
+                return True
+            time.sleep(STARTUP_POLL_SECONDS)
+
+    def _post(self, code: str, *, timeout: float) -> dict[str, object] | None:
+        """One synchronous request. A transport failure is not an answer."""
+        if timeout <= 0:
+            return None
+        payload = json.dumps({"query": code}).encode()
+        call = urllib.request.Request(
+            f"http://127.0.0.1:{self._port}/query-sync", data=payload, headers={"Content-Type": "application/json"}
+        )
+        try:
+            with urllib.request.urlopen(call, timeout=timeout) as response:  # noqa: S310 -- fixed loopback URL
+                body = json.load(response)
+        except (urllib.error.URLError, TimeoutError, OSError, ValueError):
+            self.last_body = ""
+            return None
+        if not isinstance(body, dict):
+            self.last_body = ""
+            return None
+        self.last_body = str(body.get("stdout", ""))
+        return body
+
+    def evaluate(self, code: str, *, timeout: float) -> SessionAnswer | None:
+        """Evaluate Scala in the session and return the output it printed, verified by receipt.
+
+        The evaluated code's console output is redirected into a private file whose first line is
+        a nonce this call generated. An answer is accepted only when that nonce comes back, so an
+        empty body, a cached file or another request's output cannot be read as this answer.
+        """
+        if not self.alive:
+            self._poison(self.failure or "session_not_running")
+            return None
+        remaining = self._remaining(timeout)
+        if remaining <= 0:
+            self._poison("deadline_exhausted")
+            return None
+        nonce = uuid.uuid4().hex
+        target = self.scratch / f"answer-{nonce}.txt"
+        wrapped = _redirected(code, target, nonce)
+        started = time.monotonic()
+        body = self._post(wrapped, timeout=remaining)
+        seconds = time.monotonic() - started
+        self.requests += 1
+        if body is None:
+            self._poison("session_request_failed")
+            return None
+        if body.get("success") not in (True, "true"):
+            self._poison("session_request_unsuccessful")
+            return None
+        try:
+            written = target.read_text()
+        except OSError:
+            self._poison("session_answer_missing")
+            return None
+        finally:
+            target.unlink(missing_ok=True)
+        first, _, rest = written.partition("\n")
+        if first.strip() != nonce:
+            # A body that does not carry this call's receipt is somebody else's output, or none.
+            self._poison("session_receipt_mismatch")
+            return None
+        return SessionAnswer(rest, seconds)
+
+    def define(self, code: str, *, timeout: float) -> str | None:
+        """Send code that must reach the REPL's top level, such as a definition.
+
+        A definition cannot be nested inside the receipt wrapper: `@main` and top-level
+        declarations are not block expressions, and wrapping them fails to compile. So this path
+        has no receipt, and its return value is therefore NOT proof that anything was defined.
+        The server answers `success` even for a compile error and puts the diagnostic in its body,
+        which is exactly why every real answer goes through `evaluate` and its receipt instead.
+        Returns the body the engine printed, or None when the request itself failed.
+        """
+        if not self.alive:
+            self._poison(self.failure or "session_not_running")
+            return None
+        remaining = self._remaining(timeout)
+        if remaining <= 0:
+            self._poison("deadline_exhausted")
+            return None
+        body = self._post(code, timeout=remaining)
+        self.requests += 1
+        if body is None:
+            self._poison("session_request_failed")
+            return None
+        return str(body.get("stdout", ""))
+
+    def load(self, graph: Path, *, timeout: float) -> bool:
+        """Load one graph, recording which is resident so a stale `cpg` cannot answer.
+
+        Loading a second graph replaces the first. The recorded identity is what lets the caller
+        refuse to answer a question about a graph that is not the one loaded.
+        """
+        answer = self.evaluate(f'importCpg({json.dumps(str(graph))}); "loaded"', timeout=timeout)
+        if answer is None:
+            self.loaded_graph = ""
+            return False
+        self.loaded_graph = str(graph)
+        return True
+
+    def close(self) -> None:
+        """Kill the group and reap it inside the cancellation allowance."""
+        process, self._process = self._process, None
+        if process is not None and process.poll() is None:
+            end = time.monotonic() + self.budget.cancellation_allowance_seconds
+            try:
+                os.killpg(process.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+            except OSError:
+                self._poison("session_cleanup_incomplete")
+            try:
+                process.wait(timeout=max(0.0, end - time.monotonic()))
+            except subprocess.TimeoutExpired:
+                self._poison("session_cleanup_incomplete")
+        for stream in (process.stdout if process else None, process.stderr if process else None):
+            if stream is not None:
+                stream.close()
+        if self._log is not None:
+            self._log.close()
+            self._log = None
+
+    def __enter__(self) -> EngineSession:
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.close()
+
+
+def _redirected(code: str, target: Path, nonce: str) -> str:
+    """Wrap `code` so its console output lands in `target` behind this call's receipt line.
+
+    The receipt is written before the code runs, so a body that never executed leaves a file
+    whose payload is empty rather than a file that does not exist: the two are distinguishable.
+    """
+    path = json.dumps(str(target))
+    return (
+        "{ val __ousastPath = java.nio.file.Path.of(" + path + "); "
+        "java.nio.file.Files.writeString(__ousastPath, " + json.dumps(nonce + "\n") + "); "
+        "val __ousastOut = new java.io.FileOutputStream(" + path + ", true); "
+        # `scala.Console`, fully qualified: the Joern REPL shadows `Console` with
+        # `io.joern.console.Console`, which has no `withOut`, and the resulting compile error
+        # arrives as a successful response with an empty receipt.
+        "try { scala.Console.withOut(new java.io.PrintStream(__ousastOut, true)) { " + code + " } } "
+        'finally { __ousastOut.close() }; "ok" }'
+    )