From 9c50e17c1148675dc5385a5f8c22b1da668bafde Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:19:32 +0200 Subject: [PATCH] fix(cpg): retry a failed build through the language frontend Scanning a 637-file WordPress plugin reported 1577 regions with 0 scanned and `cpg_build_failed`. The build was not slow or out of memory -- it failed in 42 seconds -- and `php2cpg` run directly on the same tree produced a 4.3MB CPG with no errors at all. The difference is that `joern-parse` detects the language and then shells out to the frontend in a separate process. On this tree it got as far as applying overlays and then threw, while still writing an unusable cpg.bin. Joern's own output recommends the direct route for a large codebase: "Invoking CPG generator in a separate process... If you are importing a large codebase (and/or running into memory issues), please try: invoke the frontend: php2cpg -J-Xmx3072m /tmp/src -o cpg.bin" So a failed `joern-parse` now retries through `2cpg`, with the driver passing the dominant region language. A language with no mapping fails closed exactly as before -- no guessing at a frontend name. This is the documented fallback rather than a workaround, and it was diagnosed rather than assumed: the standalone build was run first, and only after it succeeded on the identical tree was the retry added. Gates byte-identical. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- src/openultrasast/cpg/backend.py | 43 +++++++++++++++++++++++++++-- src/openultrasast/model/scan.py | 15 +++++++++- tests/test_cpg_backend.py | 47 ++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/src/openultrasast/cpg/backend.py b/src/openultrasast/cpg/backend.py index a3ffe6e..eeeba28 100644 --- a/src/openultrasast/cpg/backend.py +++ b/src/openultrasast/cpg/backend.py @@ -47,6 +47,21 @@ QUERY_TIMEOUT_SECONDS = 300 CPG_HEAP_MB = 2048 HEAP_ENV = "OPENULTRASAST_CPG_HEAP_MB" +# `joern-parse` detects the language and then shells out to a frontend, and on a large tree it can fail where +# the frontend alone succeeds -- measured on a 637-file WordPress plugin, where joern-parse threw after +# applying overlays while `php2cpg` on the same tree produced a 4.3MB CPG with no errors. Joern's own message +# recommends the direct route for large codebases, so this is the documented fallback rather than a +# workaround. The names are Joern's frontend executables. +_FRONTENDS = { + "php": "php2cpg", + "python": "pysrc2cpg", + "javascript": "jssrc2cpg", + "typescript": "jssrc2cpg", + "java": "javasrc2cpg", + "c": "c2cpg", + "c_cpp": "c2cpg", +} + Runner = Callable[..., subprocess.CompletedProcess[str]] @@ -116,8 +131,12 @@ class JoernBackend: return has_cpg() - def build(self, root: Path) -> CpgResult | None: - """Build a CPG for ``root``. ``None`` on any failure — the caller degrades, never guesses.""" + def build(self, root: Path, *, language: str = "") -> CpgResult | None: + """Build a CPG for ``root``. ``None`` on any failure — the caller degrades, never guesses. + + ``language`` lets a failed ``joern-parse`` retry through the frontend directly, which is what Joern + itself advises for a large codebase and what a 637-file WordPress plugin needed. + """ parse = shutil.which("joern-parse") if parse is None and os.environ.get("OPENULTRASAST_JOERN_PROBE", "").strip().lower() not in {"1", "on", "true", "yes"}: return None @@ -128,13 +147,31 @@ class JoernBackend: if completed is None or completed.returncode != 0 or not cpg_path.is_file(): detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout" logger.warning("cpg build failed for %s: %s", root, detail) - return None + if not self._build_with_frontend(root, cpg_path, scratch, language): + return None return CpgResult( cpg_path=cpg_path, run=lambda query, params: self.query(cpg_path, query, params), run_batch=lambda query, requests: self.query_batch(cpg_path, query, requests), ) + def _build_with_frontend(self, root: Path, cpg_path: Path, scratch: Path, language: str) -> bool: + """Retry the build through the language frontend alone. False when there is nothing to retry with.""" + frontend = _FRONTENDS.get(language.lower()) + if not frontend: + return False + binary = shutil.which(frontend) + if binary is None: + logger.warning("no frontend %s on PATH to retry the build for %s", frontend, root) + return False + completed = self._run([binary, self._heap_flag(), str(root), "-o", str(cpg_path)], timeout=self.build_timeout, cwd=scratch) + if completed is None or completed.returncode != 0 or not cpg_path.is_file(): + detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout" + logger.warning("%s also failed for %s: %s", frontend, root, detail) + return False + logger.info("built the cpg for %s with %s after joern-parse failed", root, frontend) + return True + def query_batch(self, cpg_path: Path, query: str, requests: Mapping[str, Mapping[str, object]]) -> dict[str, list[object]]: """Run ONE script invocation carrying many requests, keyed back to their ids. diff --git a/src/openultrasast/model/scan.py b/src/openultrasast/model/scan.py index dd72879..c787d34 100644 --- a/src/openultrasast/model/scan.py +++ b/src/openultrasast/model/scan.py @@ -110,7 +110,12 @@ def scan_repository( taxonomy = load_families() build_started = time.monotonic() - cpg = backend.build(root) + # The dominant region language, so a failed `joern-parse` can retry through that frontend directly. + counts: dict[str, int] = {} + for region in regions: + counts[region.language] = counts.get(region.language, 0) + 1 + dominant = max(counts, key=lambda name: (counts[name], name)) if counts else "" + cpg = _build(backend, root, dominant) build_seconds = round(time.monotonic() - build_started, 2) if cpg is None: return ModelScanResult( @@ -223,6 +228,14 @@ def scan_repository( ) +def _build(backend: Any, root: Path, language: str) -> Any: + """Build through the backend, passing the language when the backend can use it.""" + try: + return backend.build(root, language=language) + except TypeError: # a backend from before the retry existed, including every test double + return backend.build(root) + + def _prefetched(rows: list[object]): # type: ignore[no-untyped-def] """A `run` that ignores the question and returns rows already fetched in the batch.""" diff --git a/tests/test_cpg_backend.py b/tests/test_cpg_backend.py index 2a39831..a0f8c61 100644 --- a/tests/test_cpg_backend.py +++ b/tests/test_cpg_backend.py @@ -256,3 +256,50 @@ def test_the_operator_can_raise_the_heap(tmp_path: Path, monkeypatch: pytest.Mon JoernBackend(runner=runner).build(tmp_path) assert "-J-Xmx6144m" in commands[0] + + +def test_a_failed_joern_parse_retries_through_the_language_frontend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Measured on a 637-file WordPress plugin: `joern-parse` threw after applying overlays while `php2cpg` + on the same tree produced a 4.3MB CPG with no errors. Joern's own message recommends the direct route for + a large codebase, so this is the documented fallback rather than a workaround.""" + from openultrasast.cpg.backend import JoernBackend + + tried: list[str] = [] + + def runner(command, **kwargs): # type: ignore[no-untyped-def] + class _Result: + stderr = "" + stdout = "" + + tried.append(Path(command[0]).name) + if Path(command[0]).name == "joern-parse": + _Result.returncode = 1 + return _Result() + _Result.returncode = 0 + Path(command[command.index("-o") + 1]).write_text("cpg") + return _Result() + + monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") + monkeypatch.setattr("shutil.which", lambda name: f"/opt/joern/{name}") + + result = JoernBackend(runner=runner).build(tmp_path, language="php") + + assert result is not None, "the frontend succeeded where joern-parse did not" + assert tried == ["joern-parse", "php2cpg"] + + +def test_an_unknown_language_has_nothing_to_retry_with(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """No guessing at a frontend name: a language with no mapping fails closed, as it did before.""" + from openultrasast.cpg.backend import JoernBackend + + def runner(command, **kwargs): # type: ignore[no-untyped-def] + class _Result: + returncode = 1 + stderr = "" + stdout = "" + + return _Result() + + monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") + assert JoernBackend(runner=runner).build(tmp_path, language="cobol") is None + assert JoernBackend(runner=runner).build(tmp_path) is None