perf(cpg): run the overlay pass in-session and verify it saved (M2 11.4)

The overlay was the last fixed cost bypassing the session: a whole JVM start,
15.8 s in the measured Node runs, while the census and query batches already route
through it.

`save` writes under the process working directory, which for a session is its own
private scratch rather than the build scratch, so the saved graph is now looked for
in both. Finding neither keeps the raw graph and records it rather than reporting
success, because a graph without its dataflow layer has that layer recomputed by
every later batch, 54 s each on a 637-file plugin, with nothing otherwise saying so.

The disposable invocation remains the fallback and runs unchanged whenever the
session is off or cannot answer.

Controls cover both new paths: an engine that reports success but saves no graph
keeps the raw one and notes it, and a graph saved under the session's workspace is
found and applied.

1,395 tests passed, nine skipped. The transport stays off by default; task 11.5
measures against the frozen profile. Rollout stays NO-GO.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

src/openultrasast/cpg/backend.py

@@ -923,24 +923,42 @@ class JoernBackend:
         A failure here is logged and the raw graph kept, so the scan degrades to the old cost rather than
         losing its graph.
         """
-        joern = shutil.which("joern")
+        # M2 11.4: the overlay is a whole JVM start, 15.8 s in the measured Node runs, and it is the
-        if joern is None:
+        # last fixed cost that bypassed the session. `save` writes under the PROCESS working
-            return False
+        # directory, which for a session is its own scratch and not this one, so the saved graph is
-        script = self.queries_dir / "overlay.sc"
+        # looked for in both. Finding neither keeps the raw graph rather than reporting success:
-        command = [joern, self._heap_flag(), "--script", str(script), "--param", f"cpgFile={cpg_path}"]
+        # a graph without its dataflow layer has that layer recomputed by every later batch, 54 s
-        completed = self._run(command, timeout=self.build_timeout, cwd=scratch)
+        # each on a 637-file plugin, and nothing would otherwise say so.
-        workspace = scratch / "workspace"
+        session = self._session_for(cpg_path)
-        saved = workspace / cpg_path.name / "cpg.bin"
+        workspaces = [scratch / "workspace"]
-        try:
+        answered = False
+        if session is not None:
+            workspaces.insert(0, session.scratch / "workspace")
+            body = self._session_payload(cpg_path, "overlay", {"cpgFile": str(cpg_path)})
+            answered = body is not None and extract_payload(body) is not None
+        if not answered:
+            joern = shutil.which("joern")
+            if joern is None:
+                return False
+            script = self.queries_dir / "overlay.sc"
+            command = [joern, self._heap_flag(), "--script", str(script), "--param", f"cpgFile={cpg_path}"]
+            completed = self._run(command, timeout=self.build_timeout, cwd=scratch)
             answered = completed is not None and completed.returncode == 0 and extract_payload(completed.stdout or "") is not None
-            if not answered or not saved.is_file():
+            if not answered:
                 detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout"
                 logger.warning("overlays could not be applied to %s, keeping the raw graph: %s", cpg_path, detail)
+        saved = next((w / cpg_path.name / "cpg.bin" for w in workspaces if (w / cpg_path.name / "cpg.bin").is_file()), None)
+        try:
+            if not answered or saved is None:
+                if answered:
+                    self._note("overlay_saved_graph_missing")
+                    logger.warning("overlays reported success but saved no graph for %s, keeping the raw graph", cpg_path)
                 return False
             saved.replace(cpg_path)
             return True
         finally:
-            self._cleanup(workspace)
+            for workspace in workspaces:
+                self._cleanup(workspace)
 
     def _build_with_retries(
         self, root: Path, cpg_path: Path, scratch: Path, language: str, exclude: Sequence[str] = ()

tests/test_cpg_session.py

@@ -218,3 +218,48 @@ def test_a_session_that_cannot_start_falls_back_rather_than_answering(tmp_path):
     assert backend._session_payload(tmp_path / "cpg.bin", "census", {"cpgFile": "x"}) is None
     assert any("session_unavailable" in note or "session" in note for note in backend._diagnostics) or backend._diagnostics == []
     backend.close_session()
+
+
+def test_an_overlay_that_saves_no_graph_keeps_the_raw_one(tmp_path, monkeypatch):
+    """M2 11.4: reported success with no saved graph must not read as an applied overlay."""
+    from openultrasast.cpg.backend import JoernBackend
+
+    backend = JoernBackend(execution_budget=ExecutionBudget(time.monotonic() + 30, 1.0))
+    graph = tmp_path / "cpg.bin"
+    graph.write_bytes(b"raw")
+    scratch = tmp_path / "scratch"
+    scratch.mkdir()
+    # The engine answers, but nothing lands in any workspace.
+    monkeypatch.setattr("openultrasast.cpg.backend.shutil.which", lambda name: "/usr/bin/" + name)
+    monkeypatch.setattr(JoernBackend, "_run", lambda *a, **k: _completed('---OUSAST-CPG-BEGIN---\n{"files":"1"}\n---OUSAST-CPG-END---'))
+    assert backend._apply_overlays(graph, scratch) is False
+    assert graph.read_bytes() == b"raw"
+    assert "overlay_saved_graph_missing" in backend._diagnostics
+
+
+def test_an_overlay_saved_under_the_session_workspace_is_found(tmp_path, monkeypatch):
+    from openultrasast.cpg.backend import JoernBackend
+    from openultrasast.cpg.session import EngineSession
+
+    backend = JoernBackend(session_transport=True, execution_budget=ExecutionBudget(time.monotonic() + 30, 1.0))
+    graph = tmp_path / "cpg.bin"
+    graph.write_bytes(b"raw")
+    scratch = tmp_path / "scratch"
+    scratch.mkdir()
+    session_scratch = tmp_path / "session"
+    saved = session_scratch / "workspace" / graph.name
+    saved.mkdir(parents=True)
+    (saved / "cpg.bin").write_bytes(b"overlaid")
+    live = EngineSession(session_scratch, ExecutionBudget(time.monotonic() + 30, 1.0), {})
+    monkeypatch.setattr(JoernBackend, "_session_for", lambda self, path: live)
+    monkeypatch.setattr(
+        JoernBackend, "_session_payload", lambda self, path, query, params: '---OUSAST-CPG-BEGIN---\n{"files":"1"}\n---OUSAST-CPG-END---'
+    )
+    assert backend._apply_overlays(graph, scratch) is True
+    assert graph.read_bytes() == b"overlaid"
+
+
+def _completed(stdout):
+    import subprocess
+
+    return subprocess.CompletedProcess(["joern"], 0, stdout, "")