fix(cpg): the instrument reports on itself -- census failure fails the build, overlays and heap are checked

Three checks for three findings that were each silently wrong for a day (task 5.13). A clean build whose
census cannot be taken inside the query timeout is a failed build, by name: every later query would time
out and read as "nothing here", and the launcher fallback no longer rebuilds the same graph without
asking again. The census now carries the graph's saved overlays and the worker JVM's actual heap, and the
backend warns when the dataflow layer is missing (every batch would recompute it) or the heap is not the
configured one (the knob is not reaching the JVM that matters). An empty tree counts as complete, not as
a failed count.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/cpg/backend.py

@@ -96,6 +96,10 @@ _LANGUAGE_EXTENSIONS: dict[str, tuple[str, ...]] = {"php": (".php",)}
 # through a lookup table that a scan could name, and the file is named in the log so the exclusion is
 # visible rather than silent. Task 5.13.
 MAX_SOURCE_BYTES = 1_000_000
+
+# The overlay a graph must carry for `reachableByFlows` to be answered from the saved graph rather than
+# recomputed at every load; Joern names it so in `cpg.metaData.overlays`.
+DATAFLOW_OVERLAY = "dataflowOss"
 # Frontends that shell out to a separate interpreter, and the command that proves it can read a file.
 # `php2cpg` drives PHP-Parser through whatever `php` is on PATH, so the frontend is only as good as that
 # interpreter's view of the filesystem -- and a `php` that cannot see the tree does not fail, it produces an
@@ -274,7 +278,13 @@ class JoernBackend:
 
         # A frontend whose failures are visible and retryable is worth more than one whose are not.
         if language.lower() in _PREFER_FRONTEND:
+            self.last_failure = ""
             shards, unparsed = self._build_sharded(root, scratch, language)
+            if not shards and self.last_failure:
+                # A build that FAILED BY NAME is not one to retry through the other launcher: the graph the
+                # census could not load would be rebuilt the same and never asked again.
+                dispose()
+                return None
             if shards:
                 if unparsed:
                     logger.warning("the frontend could not parse %d file(s) under %s: %s", len(unparsed), root, ", ".join(unparsed[:5]))
@@ -431,7 +441,15 @@ class JoernBackend:
             # A build that warned about nothing is exactly the build that needs checking, because a graph
             # holding nothing arrives with no warnings at all. Silence is the symptom that has no symptom.
             census = self._graph_census(main, root, language)
-            if census is not None and census[0] < census[1] - len(defective):
+            if census is None:
+                # A graph the census cannot load inside the query timeout cannot answer a taint batch
+                # either; every one would time out and read as "nothing here". Measured: `cpg query census
+                # failed: timeout` was logged as a warning during a build whose every later query then
+                # timed out, and the warning was read as a warning. A build with no usable graph has failed.
+                self.last_failure = "the graph could not answer the census (timeout or no payload), so no query would be answered either"
+                logger.error("cpg build for %s: %s", root, self.last_failure)
+                return (), ()
+            if census[0] < census[1] - len(defective):
                 self.last_failure = f"the frontend reported no failures but the graph holds {census[0]} of {census[1]} source files"
                 logger.error("cpg build for %s: %s", root, self.last_failure)
             return (main,), defective
@@ -474,19 +492,46 @@ class JoernBackend:
             return None
         expected = sum(1 for path in root.rglob("*") if path.suffix.lower() in extensions and path.is_file())
         if not expected:
-            return None
+            return (0, 0)  # nothing to count is not a failed count: an empty tree's graph is complete
         payload = self.query(cpg_path, "census", {})
         if not isinstance(payload, Mapping):
             return None
+        self._check_instrument(payload)
         try:
             return int(str(payload.get("files", "0"))), expected
         except ValueError:
             return None
 
+    def _check_instrument(self, census: Mapping[str, object]) -> None:
+        """Two facts the census carries about the engine itself, checked because each was once silently wrong.
+
+        A graph whose saved overlays lack the dataflow layer has it recomputed by every query batch -- 54 s
+        of fixed cost per batch on a 637-file plugin, an OutOfMemoryError on a larger one -- and nothing
+        else in the pipeline can tell. A worker JVM whose heap is not the configured one means the heap
+        knob governs nothing, which was true for a day. Both are warnings, not failures: the scan still
+        runs, slower or smaller, and the log says why. Task 5.13.
+        """
+        overlays = str(census.get("overlays", ""))
+        if overlays and DATAFLOW_OVERLAY not in overlays.split(","):
+            logger.warning(
+                "the graph carries overlays [%s] but not %s: every query batch will recompute the dataflow layer",
+                overlays,
+                DATAFLOW_OVERLAY,
+            )
+        reported = str(census.get("maxHeapMB", ""))
+        if reported.isdigit():
+            configured = self._heap_mb()
+            if abs(int(reported) - configured) > configured // 4:
+                logger.warning(
+                    "the query JVM reports a %s MB heap where %d MB is configured; the heap setting is not reaching it",
+                    reported,
+                    configured,
+                )
+
     def _graph_is_complete(self, cpg_path: Path, root: Path, language: str) -> bool:
         """Conservative: when the census cannot be taken, the answer is "no" and the caller is careful."""
         census = self._graph_census(cpg_path, root, language)
-        return census is not None and census[0] >= census[1]
+        return census is not None and census[1] > 0 and census[0] >= census[1]
 
     def _island_root(self, root: Path, scratch: Path, files: Sequence[str]) -> Path | None:
         """A tree holding only ``files``, at their original relative paths.

src/openultrasast/cpg/queries/census.sc

@@ -8,11 +8,25 @@
 // This matters because acting on the warning alone is destructive: it makes the driver split a repository
 // into shards that cannot see each other's flows, and a two-file WordPress pair lost its CVE that way.
 //
+// It also reports two facts about the INSTRUMENT that were each, once, wrong for a day without anything
+// saying so (contributor-scan 5.13): which overlays the graph carries, because a graph without its
+// dataflow layer saved has it recomputed by every query batch; and the heap the worker JVM actually got,
+// because `-J-Xmx` reaches only the launcher and the configured heap governed nothing until that was seen.
+//
 // Parameters: cpgFile.
-// Output: fenced JSON {files, methods}.
+// Output: fenced JSON {files, methods, overlays, maxHeapMB}.
 @main def exec(cpgFile: String) = {
   importCpg(cpgFile)
   println("---OUSAST-CPG-BEGIN---")
-  println(ujson.write(ujson.Obj("files" -> cpg.file.size.toString, "methods" -> cpg.method.size.toString)))
+  println(
+    ujson.write(
+      ujson.Obj(
+        "files"     -> cpg.file.size.toString,
+        "methods"   -> cpg.method.size.toString,
+        "overlays"  -> cpg.metaData.overlays.l.flatten.mkString(","),
+        "maxHeapMB" -> (Runtime.getRuntime.maxMemory / 1048576).toString
+      )
+    )
+  )
   println("---OUSAST-CPG-END---")
 }

tests/test_cpg_backend.py

@@ -450,6 +450,13 @@ def test_php_builds_through_the_frontend_and_retries_while_files_are_dropped(tmp
             return _overlaid(command, kwargs.get("cwd"))
         if "-r" in command:
             return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="")
+        if "--script" in command:  # the census on the clean build
+            return subprocess.CompletedProcess(
+                args=command,
+                returncode=0,
+                stdout='---OUSAST-CPG-BEGIN---\n{"files": "1", "methods": "1"}\n---OUSAST-CPG-END---\n',
+                stderr="",
+            )
         tried.append(Path(command[0]).name)
         Path(command[command.index("-o") + 1]).write_text("cpg")
         # Drops a file on the first two attempts, clean on the third.
@@ -880,7 +887,12 @@ def test_a_frontend_build_gets_its_overlays_once_at_build_time(tmp_path: Path, m
         if _is_overlay(command):
             return _overlaid(command, kwargs.get("cwd"))
         if "--script" in command:
-            return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="")
+            return subprocess.CompletedProcess(
+                args=command,
+                returncode=0,
+                stdout='---OUSAST-CPG-BEGIN---\n{"files": "1", "methods": "1"}\n---OUSAST-CPG-END---\n',
+                stderr="",
+            )
         Path(command[command.index("-o") + 1]).write_text("raw")
         return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="")
 
@@ -941,3 +953,68 @@ def test_a_source_file_the_size_of_a_data_table_is_excluded_and_named(tmp_path:
     assert "--exclude" in build and "vendor/profiles.php" in build, "the data table is excluded from the build by name"
     assert not any("code.php" in part for part in build), "and the code is not"
     assert [Path(name).name for name in result.unparsed] == ["profiles.php"], "and it is reported, not silently dropped"
+
+
+def test_a_clean_build_whose_census_cannot_be_taken_is_a_failed_build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """`cpg query census failed: timeout` was logged as a warning during a build whose every later query
+    then timed out too. A graph that cannot answer the census inside the query timeout cannot answer a taint
+    batch either, so the build fails, by name, instead of handing back a graph that reads as clean."""
+    import subprocess
+
+    from openultrasast.cpg.backend import JoernBackend
+
+    (tmp_path / "a.php").write_text("<?php\nfunction a($x) { return $x; }\n")
+
+    def runner(command, **kwargs):  # type: ignore[no-untyped-def]
+        if _is_overlay(command):
+            return _overlaid(command, kwargs.get("cwd"))
+        if "-r" in command:
+            return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="")
+        if "--script" in command:
+            raise subprocess.TimeoutExpired(command, 1)  # the census never answers
+        if "--output" in command:  # the joern-parse fallback the backend tries after a failed frontend build
+            return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="")
+        Path(command[command.index("-o") + 1]).write_text("cpg")
+        return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="")
+
+    monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on")
+    monkeypatch.setattr("shutil.which", lambda name: f"/opt/bin/{name}")
+    backend = JoernBackend(runner=runner)
+    assert backend.build(tmp_path, language="php") is None
+    assert "census" in backend.last_failure
+
+
+def test_the_census_checks_the_instrument(caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    """Two facts about the engine, each once wrong for a day with nothing saying so: a graph without its
+    dataflow overlay saved (every batch recomputes it) and a worker JVM whose heap is not the configured one
+    (the knob governs nothing). Both are named in the log; neither fails the scan."""
+    import logging
+    import subprocess
+
+    from openultrasast.cpg.backend import BEGIN, END, JoernBackend
+
+    (tmp_path / "a.php").write_text("<?php\n")
+    monkeypatch.setenv("OPENULTRASAST_CPG_HEAP_MB", "2048")
+
+    def census(body: str):  # type: ignore[no-untyped-def]
+        def runner(command, **kwargs):  # type: ignore[no-untyped-def]
+            return subprocess.CompletedProcess(args=command, returncode=0, stdout=f"{BEGIN}\n{body}\n{END}\n", stderr="")
+
+        return runner
+
+    monkeypatch.setattr("shutil.which", lambda name: f"/opt/bin/{name}")
+    with caplog.at_level(logging.WARNING):
+        JoernBackend(
+            runner=census('{"files": "1", "methods": "1", "overlays": "base,controlflow,typerel,callgraph", "maxHeapMB": "1979"}')
+        )._graph_census(tmp_path / "c.bin", tmp_path, "php")
+    assert "not dataflowOss" in caplog.text and "recompute" in caplog.text
+    assert "heap" not in caplog.text, "1979 MB for a 2048 MB setting is the JVM's own rounding, not a missing knob"
+    caplog.clear()
+    with caplog.at_level(logging.WARNING):
+        JoernBackend(
+            runner=census(
+                '{"files": "1", "methods": "1", "overlays": "base,controlflow,typerel,callgraph,dataflowOss", "maxHeapMB": "512"}'
+            )
+        )._graph_census(tmp_path / "c.bin", tmp_path, "php")
+    assert "512 MB heap where 2048 MB is configured" in caplog.text
+    assert "recompute" not in caplog.text