diff --git a/src/openultrasast/cpg/backend.py b/src/openultrasast/cpg/backend.py index 758449c..fba30f2 100644 --- a/src/openultrasast/cpg/backend.py +++ b/src/openultrasast/cpg/backend.py @@ -96,6 +96,10 @@ _LANGUAGE_EXTENSIONS: dict[str, tuple[str, ...]] = {"php": (".php",)} # through a lookup table that a scan could name, and the file is named in the log so the exclusion is # visible rather than silent. Task 5.13. MAX_SOURCE_BYTES = 1_000_000 + +# The overlay a graph must carry for `reachableByFlows` to be answered from the saved graph rather than +# recomputed at every load; Joern names it so in `cpg.metaData.overlays`. +DATAFLOW_OVERLAY = "dataflowOss" # Frontends that shell out to a separate interpreter, and the command that proves it can read a file. # `php2cpg` drives PHP-Parser through whatever `php` is on PATH, so the frontend is only as good as that # interpreter's view of the filesystem -- and a `php` that cannot see the tree does not fail, it produces an @@ -274,7 +278,13 @@ class JoernBackend: # A frontend whose failures are visible and retryable is worth more than one whose are not. if language.lower() in _PREFER_FRONTEND: + self.last_failure = "" shards, unparsed = self._build_sharded(root, scratch, language) + if not shards and self.last_failure: + # A build that FAILED BY NAME is not one to retry through the other launcher: the graph the + # census could not load would be rebuilt the same and never asked again. + dispose() + return None if shards: if unparsed: logger.warning("the frontend could not parse %d file(s) under %s: %s", len(unparsed), root, ", ".join(unparsed[:5])) @@ -431,7 +441,15 @@ class JoernBackend: # A build that warned about nothing is exactly the build that needs checking, because a graph # holding nothing arrives with no warnings at all. Silence is the symptom that has no symptom. census = self._graph_census(main, root, language) - if census is not None and census[0] < census[1] - len(defective): + if census is None: + # A graph the census cannot load inside the query timeout cannot answer a taint batch + # either; every one would time out and read as "nothing here". Measured: `cpg query census + # failed: timeout` was logged as a warning during a build whose every later query then + # timed out, and the warning was read as a warning. A build with no usable graph has failed. + self.last_failure = "the graph could not answer the census (timeout or no payload), so no query would be answered either" + logger.error("cpg build for %s: %s", root, self.last_failure) + return (), () + if census[0] < census[1] - len(defective): self.last_failure = f"the frontend reported no failures but the graph holds {census[0]} of {census[1]} source files" logger.error("cpg build for %s: %s", root, self.last_failure) return (main,), defective @@ -474,19 +492,46 @@ class JoernBackend: return None expected = sum(1 for path in root.rglob("*") if path.suffix.lower() in extensions and path.is_file()) if not expected: - return None + return (0, 0) # nothing to count is not a failed count: an empty tree's graph is complete payload = self.query(cpg_path, "census", {}) if not isinstance(payload, Mapping): return None + self._check_instrument(payload) try: return int(str(payload.get("files", "0"))), expected except ValueError: return None + def _check_instrument(self, census: Mapping[str, object]) -> None: + """Two facts the census carries about the engine itself, checked because each was once silently wrong. + + A graph whose saved overlays lack the dataflow layer has it recomputed by every query batch -- 54 s + of fixed cost per batch on a 637-file plugin, an OutOfMemoryError on a larger one -- and nothing + else in the pipeline can tell. A worker JVM whose heap is not the configured one means the heap + knob governs nothing, which was true for a day. Both are warnings, not failures: the scan still + runs, slower or smaller, and the log says why. Task 5.13. + """ + overlays = str(census.get("overlays", "")) + if overlays and DATAFLOW_OVERLAY not in overlays.split(","): + logger.warning( + "the graph carries overlays [%s] but not %s: every query batch will recompute the dataflow layer", + overlays, + DATAFLOW_OVERLAY, + ) + reported = str(census.get("maxHeapMB", "")) + if reported.isdigit(): + configured = self._heap_mb() + if abs(int(reported) - configured) > configured // 4: + logger.warning( + "the query JVM reports a %s MB heap where %d MB is configured; the heap setting is not reaching it", + reported, + configured, + ) + def _graph_is_complete(self, cpg_path: Path, root: Path, language: str) -> bool: """Conservative: when the census cannot be taken, the answer is "no" and the caller is careful.""" census = self._graph_census(cpg_path, root, language) - return census is not None and census[0] >= census[1] + return census is not None and census[1] > 0 and census[0] >= census[1] def _island_root(self, root: Path, scratch: Path, files: Sequence[str]) -> Path | None: """A tree holding only ``files``, at their original relative paths. diff --git a/src/openultrasast/cpg/queries/census.sc b/src/openultrasast/cpg/queries/census.sc index e3933fe..e8b22ee 100644 --- a/src/openultrasast/cpg/queries/census.sc +++ b/src/openultrasast/cpg/queries/census.sc @@ -8,11 +8,25 @@ // This matters because acting on the warning alone is destructive: it makes the driver split a repository // into shards that cannot see each other's flows, and a two-file WordPress pair lost its CVE that way. // +// It also reports two facts about the INSTRUMENT that were each, once, wrong for a day without anything +// saying so (contributor-scan 5.13): which overlays the graph carries, because a graph without its +// dataflow layer saved has it recomputed by every query batch; and the heap the worker JVM actually got, +// because `-J-Xmx` reaches only the launcher and the configured heap governed nothing until that was seen. +// // Parameters: cpgFile. -// Output: fenced JSON {files, methods}. +// Output: fenced JSON {files, methods, overlays, maxHeapMB}. @main def exec(cpgFile: String) = { importCpg(cpgFile) println("---OUSAST-CPG-BEGIN---") - println(ujson.write(ujson.Obj("files" -> cpg.file.size.toString, "methods" -> cpg.method.size.toString))) + println( + ujson.write( + ujson.Obj( + "files" -> cpg.file.size.toString, + "methods" -> cpg.method.size.toString, + "overlays" -> cpg.metaData.overlays.l.flatten.mkString(","), + "maxHeapMB" -> (Runtime.getRuntime.maxMemory / 1048576).toString + ) + ) + ) println("---OUSAST-CPG-END---") } diff --git a/tests/test_cpg_backend.py b/tests/test_cpg_backend.py index 8862eef..269b9f8 100644 --- a/tests/test_cpg_backend.py +++ b/tests/test_cpg_backend.py @@ -450,6 +450,13 @@ def test_php_builds_through_the_frontend_and_retries_while_files_are_dropped(tmp return _overlaid(command, kwargs.get("cwd")) if "-r" in command: return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") + if "--script" in command: # the census on the clean build + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='---OUSAST-CPG-BEGIN---\n{"files": "1", "methods": "1"}\n---OUSAST-CPG-END---\n', + stderr="", + ) tried.append(Path(command[0]).name) Path(command[command.index("-o") + 1]).write_text("cpg") # Drops a file on the first two attempts, clean on the third. @@ -880,7 +887,12 @@ def test_a_frontend_build_gets_its_overlays_once_at_build_time(tmp_path: Path, m if _is_overlay(command): return _overlaid(command, kwargs.get("cwd")) if "--script" in command: - return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='---OUSAST-CPG-BEGIN---\n{"files": "1", "methods": "1"}\n---OUSAST-CPG-END---\n', + stderr="", + ) Path(command[command.index("-o") + 1]).write_text("raw") return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") @@ -941,3 +953,68 @@ def test_a_source_file_the_size_of_a_data_table_is_excluded_and_named(tmp_path: assert "--exclude" in build and "vendor/profiles.php" in build, "the data table is excluded from the build by name" assert not any("code.php" in part for part in build), "and the code is not" assert [Path(name).name for name in result.unparsed] == ["profiles.php"], "and it is reported, not silently dropped" + + +def test_a_clean_build_whose_census_cannot_be_taken_is_a_failed_build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """`cpg query census failed: timeout` was logged as a warning during a build whose every later query + then timed out too. A graph that cannot answer the census inside the query timeout cannot answer a taint + batch either, so the build fails, by name, instead of handing back a graph that reads as clean.""" + import subprocess + + from openultrasast.cpg.backend import JoernBackend + + (tmp_path / "a.php").write_text(" None: + """Two facts about the engine, each once wrong for a day with nothing saying so: a graph without its + dataflow overlay saved (every batch recomputes it) and a worker JVM whose heap is not the configured one + (the knob governs nothing). Both are named in the log; neither fails the scan.""" + import logging + import subprocess + + from openultrasast.cpg.backend import BEGIN, END, JoernBackend + + (tmp_path / "a.php").write_text("