fix(cpg): retry a failed build through the language frontend

Scanning a 637-file WordPress plugin reported 1577 regions with 0 scanned
and `cpg_build_failed`. The build was not slow or out of memory -- it
failed in 42 seconds -- and `php2cpg` run directly on the same tree
produced a 4.3MB CPG with no errors at all.

The difference is that `joern-parse` detects the language and then shells
out to the frontend in a separate process. On this tree it got as far as
applying overlays and then threw, while still writing an unusable cpg.bin.
Joern's own output recommends the direct route for a large codebase:

  "Invoking CPG generator in a separate process... If you are importing a
   large codebase (and/or running into memory issues), please try:
   invoke the frontend: php2cpg -J-Xmx3072m /tmp/src -o cpg.bin"

So a failed `joern-parse` now retries through `<language>2cpg`, with the
driver passing the dominant region language. A language with no mapping
fails closed exactly as before -- no guessing at a frontend name.

This is the documented fallback rather than a workaround, and it was
diagnosed rather than assumed: the standalone build was run first, and only
after it succeeded on the identical tree was the retry added.

Gates byte-identical.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/cpg/backend.py

@@ -47,6 +47,21 @@ QUERY_TIMEOUT_SECONDS = 300
 CPG_HEAP_MB = 2048
 HEAP_ENV = "OPENULTRASAST_CPG_HEAP_MB"
 
+# `joern-parse` detects the language and then shells out to a frontend, and on a large tree it can fail where
+# the frontend alone succeeds -- measured on a 637-file WordPress plugin, where joern-parse threw after
+# applying overlays while `php2cpg` on the same tree produced a 4.3MB CPG with no errors. Joern's own message
+# recommends the direct route for large codebases, so this is the documented fallback rather than a
+# workaround. The names are Joern's frontend executables.
+_FRONTENDS = {
+    "php": "php2cpg",
+    "python": "pysrc2cpg",
+    "javascript": "jssrc2cpg",
+    "typescript": "jssrc2cpg",
+    "java": "javasrc2cpg",
+    "c": "c2cpg",
+    "c_cpp": "c2cpg",
+}
+
 Runner = Callable[..., subprocess.CompletedProcess[str]]
 
 
@@ -116,8 +131,12 @@ class JoernBackend:
 
         return has_cpg()
 
-    def build(self, root: Path) -> CpgResult | None:
-        """Build a CPG for ``root``. ``None`` on any failure — the caller degrades, never guesses."""
+    def build(self, root: Path, *, language: str = "") -> CpgResult | None:
+        """Build a CPG for ``root``. ``None`` on any failure — the caller degrades, never guesses.
+
+        ``language`` lets a failed ``joern-parse`` retry through the frontend directly, which is what Joern
+        itself advises for a large codebase and what a 637-file WordPress plugin needed.
+        """
         parse = shutil.which("joern-parse")
         if parse is None and os.environ.get("OPENULTRASAST_JOERN_PROBE", "").strip().lower() not in {"1", "on", "true", "yes"}:
             return None
@@ -128,13 +147,31 @@ class JoernBackend:
         if completed is None or completed.returncode != 0 or not cpg_path.is_file():
             detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout"
             logger.warning("cpg build failed for %s: %s", root, detail)
-            return None
+            if not self._build_with_frontend(root, cpg_path, scratch, language):
+                return None
         return CpgResult(
             cpg_path=cpg_path,
             run=lambda query, params: self.query(cpg_path, query, params),
             run_batch=lambda query, requests: self.query_batch(cpg_path, query, requests),
         )
 
+    def _build_with_frontend(self, root: Path, cpg_path: Path, scratch: Path, language: str) -> bool:
+        """Retry the build through the language frontend alone. False when there is nothing to retry with."""
+        frontend = _FRONTENDS.get(language.lower())
+        if not frontend:
+            return False
+        binary = shutil.which(frontend)
+        if binary is None:
+            logger.warning("no frontend %s on PATH to retry the build for %s", frontend, root)
+            return False
+        completed = self._run([binary, self._heap_flag(), str(root), "-o", str(cpg_path)], timeout=self.build_timeout, cwd=scratch)
+        if completed is None or completed.returncode != 0 or not cpg_path.is_file():
+            detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout"
+            logger.warning("%s also failed for %s: %s", frontend, root, detail)
+            return False
+        logger.info("built the cpg for %s with %s after joern-parse failed", root, frontend)
+        return True
+
     def query_batch(self, cpg_path: Path, query: str, requests: Mapping[str, Mapping[str, object]]) -> dict[str, list[object]]:
         """Run ONE script invocation carrying many requests, keyed back to their ids.
 

src/openultrasast/model/scan.py

@@ -110,7 +110,12 @@ def scan_repository(
 
     taxonomy = load_families()
     build_started = time.monotonic()
-    cpg = backend.build(root)
+    # The dominant region language, so a failed `joern-parse` can retry through that frontend directly.
+    counts: dict[str, int] = {}
+    for region in regions:
+        counts[region.language] = counts.get(region.language, 0) + 1
+    dominant = max(counts, key=lambda name: (counts[name], name)) if counts else ""
+    cpg = _build(backend, root, dominant)
     build_seconds = round(time.monotonic() - build_started, 2)
     if cpg is None:
         return ModelScanResult(
@@ -223,6 +228,14 @@ def scan_repository(
     )
 
 
+def _build(backend: Any, root: Path, language: str) -> Any:
+    """Build through the backend, passing the language when the backend can use it."""
+    try:
+        return backend.build(root, language=language)
+    except TypeError:  # a backend from before the retry existed, including every test double
+        return backend.build(root)
+
+
 def _prefetched(rows: list[object]):  # type: ignore[no-untyped-def]
     """A `run` that ignores the question and returns rows already fetched in the batch."""
 

tests/test_cpg_backend.py

@@ -256,3 +256,50 @@ def test_the_operator_can_raise_the_heap(tmp_path: Path, monkeypatch: pytest.Mon
     JoernBackend(runner=runner).build(tmp_path)
 
     assert "-J-Xmx6144m" in commands[0]
+
+
+def test_a_failed_joern_parse_retries_through_the_language_frontend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Measured on a 637-file WordPress plugin: `joern-parse` threw after applying overlays while `php2cpg`
+    on the same tree produced a 4.3MB CPG with no errors. Joern's own message recommends the direct route for
+    a large codebase, so this is the documented fallback rather than a workaround."""
+    from openultrasast.cpg.backend import JoernBackend
+
+    tried: list[str] = []
+
+    def runner(command, **kwargs):  # type: ignore[no-untyped-def]
+        class _Result:
+            stderr = ""
+            stdout = ""
+
+        tried.append(Path(command[0]).name)
+        if Path(command[0]).name == "joern-parse":
+            _Result.returncode = 1
+            return _Result()
+        _Result.returncode = 0
+        Path(command[command.index("-o") + 1]).write_text("cpg")
+        return _Result()
+
+    monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on")
+    monkeypatch.setattr("shutil.which", lambda name: f"/opt/joern/{name}")
+
+    result = JoernBackend(runner=runner).build(tmp_path, language="php")
+
+    assert result is not None, "the frontend succeeded where joern-parse did not"
+    assert tried == ["joern-parse", "php2cpg"]
+
+
+def test_an_unknown_language_has_nothing_to_retry_with(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """No guessing at a frontend name: a language with no mapping fails closed, as it did before."""
+    from openultrasast.cpg.backend import JoernBackend
+
+    def runner(command, **kwargs):  # type: ignore[no-untyped-def]
+        class _Result:
+            returncode = 1
+            stderr = ""
+            stdout = ""
+
+        return _Result()
+
+    monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on")
+    assert JoernBackend(runner=runner).build(tmp_path, language="cobol") is None
+    assert JoernBackend(runner=runner).build(tmp_path) is None