diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index fd1d5bf..101c488 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -830,6 +830,94 @@ that was 88% tier-0 pairs answering instantly. With those pruned (flow-aware-ranking phase 1), the 339 requests that CAN have an answer still exceed 2,400 s: **7–10 s per real request at `callDepth = 3`**. The memoisation was real and stays; the per-request figure to design against is that one. + - **Worse than that, measured directly.** Forty of the 339 real requests, on their own, at `callDepth = 3`: + **no payload after 57 minutes -- more than 85 s per request.** The 7–10 s figure above was itself an + average diluted by cheaper pairs. At that cost the phase-1 scan's 339 requests need roughly eight hours, + which is why 2,400 s was never close. The depth-1 and depth-0 comparison did NOT run (the harness died + after the depth-3 timeout). + - **Attributed, then removed.** Ten real pairs (file-scope regions with 1 to 346 sinks, and two functions), + `callDepth = 1`, each half switched separately, hard process-group-killing timeouts, 2 GB JVM. The fixed + cost of one Joern invocation -- boot, script compile, graph load, an EMPTY batch -- is **54 s**, and is + subtracted below: + + | configuration | 10 requests | net per request | + |---|---|---| + | both halves off | 165.0 s | 11.1 s | + | field half only | 163.4 s | 10.9 s | + | hook half only | **timeout at 540 s** | > 49 s | + | hook half only, seeds scoped to the callback's file | 151.5 s | 9.8 s | + | both halves on, after the fix | 150.8 s | 9.7 s | + + The field half costs nothing measurable. The hook half was the whole difference between the cheap REST + handlers and the real pairs, and stage timers on stderr said exactly where: the last stage to report was + `applies=5`, and the next -- `taintedKeysOf` -- never did, inside 270 s, for a single request. Its seeds + were `hookSeedsIn("")`: EVERY field access in EVERY one of the 7,061 methods, each put through the field + join's `reachableByFlows`. Once per batch, memoised, and never finishing. The seeds a callback's key + assignments can be fed by are the reads in its own body and the assignments in its own class, so the + seeds are now computed from the callback method and its file. WP Statistics -- the case the hook half + exists for -- has `set_current_page` and `$this->rest_hits = ...` in the same file and only `record` in + another, so the join should be unchanged for it. **Not yet re-verified**, and the reason is its own + finding: on the WP Statistics graph as the backend builds it today (php2cpg, one file excluded, 9.6 MB), + `importCpg` alone -- no request at all -- does not return inside 400 s, where the pmpro graph (4.2 MB) + loads in 54 s. The pre-change query times out on the same request too, so it is not the seed change; + and the backend's own census had already said `cpg query census failed: timeout` during the build, + which is the instrument reporting exactly this and being read as a warning. The suspect is the + default-overlay pass `importCpg` runs on a raw frontend graph every time -- `joern-parse` runs it once + and saves the result, php2cpg output does not -- and the measurement in flight is `importCpg` plus + `save` with a 25-minute cap. **Confirmed on the first try, at the default heap:** `importCpg` plus + `save` died of `OutOfMemoryError` inside `OssDataFlow.create` -- the reaching-definitions overlay -- + after 21 minutes. And the history closes the case: the hook half was verified at 20:42 on 09-09 + (d6908f1) from a `joern-parse` build, which applies the overlays once and saves them; at 21:11 the same + evening (1ce7e9d) PHP builds switched to raw `php2cpg` output, which has none. Every query batch since + has recomputed the whole dataflow layer before its first request -- that is the 54 s "fixed cost" on + pmpro, and on WP Statistics it is the wall. The build now runs `joern-parse --overlaysonly` once after + the frontend, so a query loads a graph that already has its layers; whether the WP Statistics pass + completes at all with a heap the product can now actually set was the next measurement: **at 4 GB it + did not OOM and did not finish either, killed at 25 minutes.** So it is not the heap, it is the pass: + something in that graph makes per-method reaching definitions explode. `--max-num-def`, the + per-method cap on that pass, did not save it either: at 1000 it skipped 11 methods and OOMed at 3 GB + after 474 s; at 200 it skipped 110 and OOMed after 432 s, both inside `initGen`. The cap counts + DEFINITIONS, and the culprits are two vendored browser-profile tables -- + `includes/vendor/whichbrowser/parser/data/profiles.php` (1.6 MB) and `models-android.php` (1.5 MB) -- + each a single array literal: one definition, a hundred thousand elements. They were 60% of the graph + (9.6 MB with them, 3.6 MB without). + - **Resolved, all three, and re-verified.** + + | | before | after | + |---|---|---| + | WP Statistics graph | 9.6 MB, overlay pass never finishes | 3.6 MB raw, overlays in 64 s once, **reloads in 8 s** | + | `record` injection request, `callDepth = 3` | timeout at 520 s | **39 s including the JVM** | + | CVE-2022-25148 at `pages.php:225` | unverifiable | **entailed, `sourceKind = hook`, unsanitized row present** | + | pmpro, 10 real pairs, both halves on | 150.8 s (raw graph) | 123.0 s (saved overlays); same 4 evidence keys, 40 duplicate flow rows fewer on one request | + + Three changes, each named in the log rather than silent: (1) `queries/overlay.sc` runs `importCpg` + + `save` once at build time and the saved graph replaces the raw one (`joern-parse --overlaysonly` is the + obvious tool and NPEs in 4.0.623); (2) a source file over `MAX_SOURCE_BYTES` (1 MB) is excluded from + the build as a data table and reported in `unparsed`; (3) the hook half's seeds are the callback's own + file. The payload diff between a raw and a saved-overlay pmpro graph was checked request by request: + identical (sink, line, source kind, sanitized) sets, fewer repeated flow paths on the saved one. + - **Open, and the maintainer's call, not mine:** `includes/vendor/` is 207 of WP Statistics's 357 PHP + files and is not excluded by `preprocess.IGNORED_DIRS`, so vendored third-party code is both scanned + and in the graph. Whether a project's vendored code is in scope is policy; the size guard above is + not, it is a tool that cannot process a lookup table and says so. + - **Also found on the way.** `sourceNodes` was a `def` composed of four `def`s, and it was consumed once + per SINK: a 346-sink file-scope region re-ran the repository-wide `frameworkSources` scan 346 times. + The four are materialised once per request now. It was not the dominant term (request 4, one sink, was + 0.4 s of graph work either way) but it was pure waste. + - **Two more instrument findings from the same afternoon.** (1) A `joern --script` run is TWO JVMs: a + launcher, which is all `-J-Xmx` reaches (163 MB resident), and a forked worker that runs the script with + no `-Xmx` at all (2.26 GB resident -- the JVM default of a quarter of physical memory). So + `OPENULTRASAST_CPG_HEAP_MB` governed nothing on this 8 GB machine and would silently take 16 GB on a + 64 GB one. The backend now also sets `JAVA_TOOL_OPTIONS`, which every JVM reads, forked ones included. + (2) The backend's own unit tests, which feed `build()` a fake engine, left fifteen `ousast-cpg-*` + directories in `/tmp` per run -- small, but the product's stale sweep waits six hours and the + directories looked exactly like the leak that was just fixed. `conftest.py` now points `tempfile` at + the test's own directory. + - **What "11 s per request" is, then.** With the hook half fixed, the real pairs cost about 10 s each at + depth 1, of which stage timers attribute well under a second to source materialisation and the joins; + the rest is `reachableByFlows` per sink, which is the query's actual work. So the 339 phase-1 requests + are roughly 55 minutes at depth 1 -- affordable as a batch, still not as a per-push scan, and the depth + question (this at `callDepth = 3`) is still the open measurement. - **The estimate this replaces was wrong and worth recording as wrong.** ">0.94s per request" came from assuming taint COMPLETED within its 2,400s ceiling. It timed out, so that was a floor presented as a figure. The measured cost is seven times higher. diff --git a/src/openultrasast/cpg/backend.py b/src/openultrasast/cpg/backend.py index c0979cd..758449c 100644 --- a/src/openultrasast/cpg/backend.py +++ b/src/openultrasast/cpg/backend.py @@ -87,6 +87,15 @@ _FRONTENDS = { _PREFER_FRONTEND = frozenset({"php"}) # The extensions that count as source for a language, used to ask whether a graph is missing anything. _LANGUAGE_EXTENSIONS: dict[str, tuple[str, ...]] = {"php": (".php",)} + +# A source file above this size is a data table, not code: WP Statistics carries two vendored browser +# profile files of 1.6 MB and 1.5 MB, each one array literal, and they were 60% of the graph (9.6 MB with +# them, 3.6 MB without). The reaching-definitions overlay does not merely slow down on them, it never +# finishes -- 25 minutes at 4 GB, OutOfMemoryError inside `initGen` at every cap `--max-num-def` offers, +# because a single assignment of a hundred-thousand-element literal is ONE definition. Nothing flows +# through a lookup table that a scan could name, and the file is named in the log so the exclusion is +# visible rather than silent. Task 5.13. +MAX_SOURCE_BYTES = 1_000_000 # Frontends that shell out to a separate interpreter, and the command that proves it can read a file. # `php2cpg` drives PHP-Parser through whatever `php` is on PATH, so the frontend is only as good as that # interpreter's view of the filesystem -- and a `php` that cannot see the tree does not fail, it produces an @@ -287,6 +296,7 @@ class JoernBackend: retried = self._build_with_frontend(root, cpg_path, scratch, language) if retried is None: return None + self._apply_overlays(cpg_path, scratch) unparsed = retried if unparsed: logger.warning("the frontend could not parse %d file(s) under %s: %s", len(unparsed), root, ", ".join(unparsed[:5])) @@ -298,6 +308,22 @@ class JoernBackend: cleanup=dispose, ) + def _files_too_large_to_flow(self, root: Path, language: str) -> tuple[str, ...]: + """Source files over `MAX_SOURCE_BYTES`, relative to the root, in the language the build is about.""" + extensions = _LANGUAGE_EXTENSIONS.get(language.lower()) + if not extensions: + return () + found: list[str] = [] + for path in sorted(root.rglob("*")): + if path.suffix.lower() not in extensions or not path.is_file(): + continue + try: + if path.stat().st_size > MAX_SOURCE_BYTES: + found.append(str(path.relative_to(root))) + except OSError: + continue + return tuple(found) + def _files_with_frontend_defect(self, root: Path, language: str) -> tuple[str, ...]: """Files carrying a construct this frontend miscompiles, found by reading the source. @@ -386,6 +412,16 @@ class JoernBackend: root, ", ".join(Path(name).name for name in defective[:5]), ) + oversized = self._files_too_large_to_flow(root, language) + if oversized: + logger.warning( + "excluding %d file(s) under %s over %d bytes (data tables the dataflow overlay never finishes on): %s", + len(oversized), + root, + MAX_SOURCE_BYTES, + ", ".join(f"{Path(name).name} ({(root / name).stat().st_size // 1024} KB)" for name in oversized[:5]), + ) + defective = (*defective, *oversized) main = scratch / "cpg.bin" dropped = self._build_with_retries(root, main, scratch, language, exclude=defective) @@ -522,8 +558,52 @@ class JoernBackend: merged["__census__"] = [{"methods": str(methods), "files": str(files), "shards": str(len(shards))}] return merged + def _apply_overlays(self, cpg_path: Path, scratch: Path) -> bool: + """Run Joern's default overlays ONCE, at build time, and keep the result in the graph. + + A frontend writes a raw graph. Everything that answers a dataflow question -- the call graph, the + type layer, the reaching-definitions pass `OssDataFlow` -- is an OVERLAY, and `importCpg` computes + every overlay the graph lacks each time a JVM opens it. `joern-parse` applies them once and saves; + `php2cpg` output has none, so from the day PHP builds went through the frontend directly + (2026-09-09, 1ce7e9d) every query batch has paid the whole pass again before its first request. + Measured: 54 s of fixed cost per batch on a 637-file plugin, and on WP Statistics the pass alone + ran the default 2 GB heap out of memory after 21 minutes -- a graph the same tool had entailed + CVE-2022-25148 on, one hour before the switch, from a `joern-parse` build. Task 5.13. + + `queries/overlay.sc` is that once: `importCpg`, `save`, and the saved graph replaces the raw one + under the same path. (`joern-parse --overlaysonly` is the obvious tool and is broken in 4.0.623.) + A failure here is logged and the raw graph kept, so the scan degrades to the old cost rather than + losing its graph. + """ + joern = shutil.which("joern") + if joern is None: + return False + script = self.queries_dir / "overlay.sc" + command = [joern, self._heap_flag(), "--script", str(script), "--param", f"cpgFile={cpg_path}"] + completed = self._run(command, timeout=self.build_timeout, cwd=scratch) + workspace = scratch / "workspace" + saved = workspace / cpg_path.name / "cpg.bin" + try: + answered = completed is not None and completed.returncode == 0 and extract_payload(completed.stdout or "") is not None + if not answered or not saved.is_file(): + detail = (completed.stderr or completed.stdout or "")[-400:] if completed is not None else "timeout" + logger.warning("overlays could not be applied to %s, keeping the raw graph: %s", cpg_path, detail) + return False + saved.replace(cpg_path) + return True + finally: + shutil.rmtree(workspace, ignore_errors=True) + def _build_with_retries( self, root: Path, cpg_path: Path, scratch: Path, language: str, exclude: Sequence[str] = () + ) -> tuple[str, ...] | None: + outcome = self._build_with_retries_raw(root, cpg_path, scratch, language, exclude) + if outcome is not None: + self._apply_overlays(cpg_path, scratch) + return outcome + + def _build_with_retries_raw( + self, root: Path, cpg_path: Path, scratch: Path, language: str, exclude: Sequence[str] = () ) -> tuple[str, ...] | None: """Build through the frontend, trying again while it reports files it could not parse. @@ -657,12 +737,30 @@ class JoernBackend: return None return extract_payload(completed.stdout or "") - def _heap_flag(self) -> str: - """``-J-Xmx``: both launchers forward ``-J`` arguments to the JVM.""" + def _heap_mb(self) -> int: if self.heap_mb > 0: - return f"-J-Xmx{self.heap_mb}m" + return self.heap_mb configured = os.environ.get(HEAP_ENV, "").strip() - return f"-J-Xmx{configured if configured.isdigit() else CPG_HEAP_MB}m" + return int(configured) if configured.isdigit() else CPG_HEAP_MB + + def _heap_flag(self) -> str: + """``-J-Xmx``: both launchers forward ``-J`` arguments to the JVM they start.""" + return f"-J-Xmx{self._heap_mb()}m" + + def _jvm_env(self) -> dict[str, str]: + """The heap for the JVM that does the work, which is NOT the one ``-J`` reaches. + + ``joern --script`` starts a launcher JVM and forks a second one to run the script + (``replpp.scripting.NonForkingScriptRunner``, despite the name), and the ``-J-Xmx`` goes to the + launcher only: 163 MB resident, next to a worker with no ``-Xmx`` at all sitting at 2.26 GB -- the + JVM default of a quarter of physical memory. So ``OPENULTRASAST_CPG_HEAP_MB`` governed nothing on an + 8 GB machine and would silently take 16 GB on a 64 GB one. ``JAVA_TOOL_OPTIONS`` is read by every + JVM at start-up, forked ones included, and is appended to rather than replaced so an operator's own + options survive. + """ + env = dict(os.environ) + env["JAVA_TOOL_OPTIONS"] = f"{env.get('JAVA_TOOL_OPTIONS', '').strip()} -Xmx{self._heap_mb()}m".strip() + return env def _run(self, command: list[str], *, timeout: int, cwd: Path | None = None) -> subprocess.CompletedProcess[str] | None: """Run one Joern command, and on timeout kill the whole PROCESS GROUP rather than the child. @@ -689,6 +787,7 @@ class JoernBackend: stderr=subprocess.PIPE, text=True, cwd=str(cwd) if cwd else None, + env=self._jvm_env(), start_new_session=True, # its own process group, so the JVM dies with the script ) as process: try: diff --git a/src/openultrasast/cpg/queries/overlay.sc b/src/openultrasast/cpg/queries/overlay.sc new file mode 100644 index 0000000..9acf24b --- /dev/null +++ b/src/openultrasast/cpg/queries/overlay.sc @@ -0,0 +1,27 @@ +// Apply Joern's default overlays ONCE, at build time, and save them into the graph (contributor-scan 5.13). +// +// A frontend writes a raw graph. The call graph, the type layer and the reaching-definitions pass that +// every dataflow question runs on are OVERLAYS, and `importCpg` computes each one the graph lacks every +// time a JVM opens it. `joern-parse` applies them once and saves; a frontend run directly does not. So +// from the day PHP builds went through php2cpg (2026-09-09, 1ce7e9d) every query batch recomputed the whole +// dataflow layer before its first request: 54 s of fixed cost on a 637-file plugin, and on WP Statistics a +// pass that ran the heap out after 21 minutes -- a graph the same tool had entailed CVE-2022-25148 on, an +// hour before the switch, from a joern-parse build. +// +// Measured after: the saved graph reloads in 8 s where the raw one took 64 s, and the request that had +// timed out at 520 s answered in 39 s including the JVM. `joern-parse --overlaysonly` would be the +// obvious tool and is broken in 4.0.623 (NullPointerException in a post-processing step that needs the +// frontend it did not run), so the once is this script. +// +// `save` writes to `workspace//cpg.bin` under the working directory; the driver moves that over +// the raw graph. +// +// Parameters: cpgFile. +// Output: fenced JSON {files, methods}. +@main def exec(cpgFile: String) = { + importCpg(cpgFile) + save + println("---OUSAST-CPG-BEGIN---") + println(ujson.write(ujson.Obj("files" -> cpg.file.size.toString, "methods" -> cpg.method.size.toString))) + println("---OUSAST-CPG-END---") +} diff --git a/src/openultrasast/cpg/queries/taint.sc b/src/openultrasast/cpg/queries/taint.sc index 772ef4b..ae3ff33 100644 --- a/src/openultrasast/cpg/queries/taint.sc +++ b/src/openultrasast/cpg/queries/taint.sc @@ -354,23 +354,21 @@ // every registered callback "return tainted" and the join degenerates into the complete graph this design // exists to refuse. Only a real framework source, or a field already shown to hold one, counts. // - // They are also NOT scoped to the requesting file, which the field half is. A hook registry is global by - // construction: the whole point of the edge is that `set_current_page` lives in one file and the - // `apply_filters` that calls it lives in another. Scoping these seeds the way the field half scopes its - // own defeats exactly the join being built, and the query returned nothing until this was separated. - val hookSeedsByFile = scala.collection.mutable.Map.empty[String, List[io.shiftleft.codepropertygraph.generated.nodes.CfgNode]] - - def hookSeedsIn(fileName: String) = - hookSeedsByFile.getOrElseUpdate( - fileName, { - val methods = methodsIn(fileName) - val framework: List[io.shiftleft.codepropertygraph.generated.nodes.CfgNode] = - methods.flatMap(_.ast.isCall.filter(c => sourcePatterns.exists(p => c.code.contains(p))).l) - val fields: List[io.shiftleft.codepropertygraph.generated.nodes.CfgNode] = - methods.flatMap(_.ast.isCall.nameExact(FIELD_ACCESS).l).filter(r => taintedPrefixes(r.code.trim, fileName)) - framework ++ fields - } - ) + // They are NOT scoped to the requesting file, which the field half is: a hook registry is global by + // construction, `set_current_page` lives in one file and the `apply_filters` that calls it in another, + // and the query returned nothing while they were. They ARE scoped to the CALLBACK's file. The first + // version scoped them to nothing -- every field access in every method of the repository, each put through + // the field join -- and on a 637-file plugin that step alone did not finish in 270 s, for a callback whose + // sources are, by the join's own logic, the reads in its body and the assignments in its class. WP + // Statistics is the case that has to survive: the callback and `$this->rest_hits = ...` share a file, + // and only the sink is elsewhere. Task 5.13 has the numbers. + def hookSeedsOf(callback: io.shiftleft.codepropertygraph.generated.nodes.Method) = { + val framework: List[io.shiftleft.codepropertygraph.generated.nodes.CfgNode] = + callback.ast.isCall.filter(c => sourcePatterns.exists(p => c.code.contains(p))).l + val fields: List[io.shiftleft.codepropertygraph.generated.nodes.CfgNode] = + callback.ast.isCall.nameExact(FIELD_ACCESS).l.filter(r => taintedPrefixes(r.code.trim, callback.filename)) + framework ++ fields + } // Half one is asked PER ARRAY KEY, not per callback, and that distinction is the whole result. // @@ -385,39 +383,40 @@ // and not a detection. The key is a literal at both ends -- written `"id"` in the callback and `['id']` // at the sink -- so it joins the same way the hook name and the field name do. Third use of one idea. val INDEX_ACCESS = ".indexAccess" - val hookKeysMemo = scala.collection.mutable.Map.empty[(String, String), Set[String]] + val hookKeysMemo = scala.collection.mutable.Map.empty[String, Set[String]] def keyOf(access: io.shiftleft.codepropertygraph.generated.nodes.Call): String = access.argument.l.lift(1).map(a => unquote(a.code)).getOrElse("") - def taintedKeysOf(fileName: String, callback: String): Set[String] = + def taintedKeysOf(callback: String): Set[String] = hookKeysMemo.getOrElseUpdate( - (fileName, callback), { - val seeds = hookSeedsIn(fileName) - if (seeds.isEmpty) Set.empty[String] - else - cpg.method - .nameExact(callback) - .ast - .isCall - .nameExact(ASSIGNMENT) - .l - .flatMap { assignment => - val args = assignment.argument.l - if (args.size < 2) None - else - args.head match { - case target: io.shiftleft.codepropertygraph.generated.nodes.Call if target.name == INDEX_ACCESS => - val key = keyOf(target) - if (key.isEmpty) None - else if (args(1).start.reachableByFlows(seeds.iterator).l.exists(f => !f.elements.l.exists(sanitizesHere))) - Some(key) - else None - case _ => None - } - } - .toSet - } + callback, + cpg.method + .nameExact(callback) + .l + .flatMap { method => + val seeds = hookSeedsOf(method) + if (seeds.isEmpty) Nil + else + method.ast.isCall + .nameExact(ASSIGNMENT) + .l + .flatMap { assignment => + val args = assignment.argument.l + if (args.size < 2) None + else + args.head match { + case target: io.shiftleft.codepropertygraph.generated.nodes.Call if target.name == INDEX_ACCESS => + val key = keyOf(target) + if (key.isEmpty) None + else if (args(1).start.reachableByFlows(seeds.iterator).l.exists(f => !f.elements.l.exists(sanitizesHere))) + Some(key) + else None + case _ => None + } + } + } + .toSet ) def hookSourceNodes = @@ -428,7 +427,7 @@ else { val keys = applies.flatMap { call => val hook = call.argument.l.headOption.map(a => unquote(a.code)).getOrElse("") - hookTable.getOrElse(hook, Nil).flatMap(cb => taintedKeysOf("", cb)) + hookTable.getOrElse(hook, Nil).flatMap(taintedKeysOf) }.toSet if (keys.isEmpty) Iterator.empty else @@ -443,7 +442,16 @@ } } - def sourceNodes = frameworkSources.l.iterator ++ parameterNodes ++ fieldSourceNodes ++ hookSourceNodes + // Materialised ONCE per request, because every one of the four is a `def` that walks the graph -- + // `frameworkSources` scans every call in the repository, `hookSourceNodes` runs one `reachableBy` per + // candidate index access -- and `sourceNodes` is consumed once per SINK. A file-scope region with 346 sinks + // paid for 346 repository scans and 346 hook joins. Measured on ten real pmpro pairs at callDepth 1: + // 16.5 s per request with the hook half off, past 54 s with it on, before this; see task 5.13. + lazy val frameworkList = frameworkSources.l + lazy val parameterList = parameterNodes.toList + lazy val fieldList = fieldSourceNodes.toList + lazy val hookList = hookSourceNodes.toList + def sourceNodes = frameworkList.iterator ++ parameterList.iterator ++ fieldList.iterator ++ hookList.iterator // WHICH KIND of source a flow started from, reported per row so the driver can tell them apart. // @@ -453,9 +461,9 @@ // present, which is right for parameters and wrong for these: on WP Statistics, seven sanitized // `$_SERVER["REQUEST_URI"]` flows displaced the unsanitized hook flow carrying CVE-2022-25148, and the // finding disappeared between the query and the verdict. - lazy val frameworkIds = frameworkSources.l.map(_.id).toSet - lazy val fieldIds = fieldSourceNodes.map(_.id).toSet - lazy val hookIds = hookSourceNodes.map(_.id).toSet + lazy val frameworkIds = frameworkList.map(_.id).toSet + lazy val fieldIds = fieldList.map(_.id).toSet + lazy val hookIds = hookList.map(_.id).toSet def sourceKindOf(head: Option[io.shiftleft.codepropertygraph.generated.nodes.AstNode]): String = head match { diff --git a/tests/conftest.py b/tests/conftest.py index 4d65793..9562d7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,3 +49,16 @@ def assert_cold_of_harnessx(): # type: ignore[no-untyped-def] return proc.stdout return _check + + +@pytest.fixture(autouse=True) +def _scratch_under_tmp_path(monkeypatch: pytest.MonkeyPatch, tmp_path_factory: pytest.TempPathFactory) -> None: + """Every CPG build makes an `ousast-cpg-*` scratch directory, and a test that feeds it a fake engine never + reaches the cleanup a real run does: one run of the backend tests left fifteen of them in the system + temp directory, and the product's stale sweep waits six hours. Pointing `tempfile` at the test's own + directory keeps them out of `/tmp` entirely, whatever the test forgets. + """ + import tempfile + + scratch = tmp_path_factory.mktemp("scratch") + monkeypatch.setattr(tempfile, "tempdir", str(scratch)) diff --git a/tests/test_cpg_backend.py b/tests/test_cpg_backend.py index f3ad8f9..8862eef 100644 --- a/tests/test_cpg_backend.py +++ b/tests/test_cpg_backend.py @@ -15,6 +15,28 @@ from pathlib import Path import pytest +def _is_overlay(command) -> bool: # type: ignore[no-untyped-def] + return any(str(part).endswith("overlay.sc") for part in command) + + +def _overlaid(command, cwd=None): # type: ignore[no-untyped-def] + """The build-time overlay step (`joern --script overlay.sc --param cpgFile=...`, run in the scratch dir), + answered the way Joern answers it: a saved graph under `workspace//cpg.bin` and a fenced payload. + + Returned by a fake runner whenever it sees the script, so tests that count frontend invocations or + dispatch on `-o` / `--script` keep counting only what they are about. + """ + import subprocess + + from openultrasast.cpg.backend import BEGIN, END + + cpg = Path(next(p for p in command if str(p).startswith("cpgFile=")).split("=", 1)[1]) + saved = Path(cwd or cpg.parent) / "workspace" / cpg.name / "cpg.bin" + saved.parent.mkdir(parents=True, exist_ok=True) + saved.write_text("cpg+overlays") + return subprocess.CompletedProcess(args=command, returncode=0, stdout=f'{BEGIN}\n{{"files": "1", "methods": "1"}}\n{END}\n', stderr="") + + def test_without_joern_the_resolved_backend_is_the_null_one(monkeypatch: pytest.MonkeyPatch) -> None: from openultrasast.cpg.backend import NullBackend, resolve_cpg_backend @@ -128,6 +150,8 @@ def test_a_batch_is_one_invocation_carrying_many_requests(monkeypatch: pytest.Mo stdout = f'{BEGIN}\n{{"r1": [{{"sink": "os.system(x)"}}], "r2": []}}\n{END}\n' def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) calls.append(list(command)) return _Done() @@ -185,6 +209,9 @@ def test_a_built_cpg_offers_the_batch_path_to_its_driver(tmp_path: Path, monkeyp scripts: list[list[str]] = [] def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) + class _Done: returncode = 0 stderr = "" @@ -220,6 +247,9 @@ def test_the_engine_runs_under_a_bounded_heap(tmp_path: Path, monkeypatch: pytes commands: list[list[str]] = [] def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) + class _Done: returncode = 0 stderr = "" @@ -248,6 +278,9 @@ def test_the_operator_can_raise_the_heap(tmp_path: Path, monkeypatch: pytest.Mon commands: list[list[str]] = [] def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) + class _Done: returncode = 0 stderr = "" @@ -278,6 +311,9 @@ def test_a_failed_joern_parse_retries_through_the_language_frontend(tmp_path: Pa tried: list[str] = [] def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) + class _Result: stderr = "" stdout = "" @@ -304,6 +340,9 @@ def test_an_unknown_language_has_nothing_to_retry_with(tmp_path: Path, monkeypat from openultrasast.cpg.backend import JoernBackend def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) + class _Result: returncode = 1 stderr = "" @@ -334,6 +373,8 @@ def test_a_batch_goes_through_a_file_not_the_command_line(tmp_path: Path) -> Non stdout = f'{BEGIN}\n{{"r0": []}}\n{END}\n' def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) commands.append(list(command)) return _Done() @@ -405,6 +446,8 @@ def test_php_builds_through_the_frontend_and_retries_while_files_are_dropped(tmp warning = "WARN AstCreationPass Failed to process '/src/big.php'\n" def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) if "-r" in command: return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") tried.append(Path(command[0]).name) @@ -433,6 +476,8 @@ def test_a_php_build_that_never_stops_dropping_files_reports_them(tmp_path: Path def runner(command, **kwargs): # type: ignore[no-untyped-def] nonlocal attempts + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) if "-r" in command: return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") attempts += 1 @@ -462,6 +507,8 @@ def test_files_the_frontend_refuses_get_a_cpg_of_their_own(tmp_path: Path, monke commands: list[list[str]] = [] def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) commands.append(list(command)) if "-r" in command: return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") # the input probe @@ -543,6 +590,8 @@ def test_a_warning_about_a_file_the_graph_actually_holds_does_not_split_it(tmp_p commands: list[list[str]] = [] def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) commands.append(list(command)) if "-r" in command: return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") # the input probe @@ -581,6 +630,8 @@ def test_a_build_is_refused_when_the_interpreter_cannot_read_the_tree(tmp_path: built: list[str] = [] def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) if "-r" in command: return subprocess.CompletedProcess(args=command, returncode=3, stdout="", stderr="") # cannot read built.append(Path(command[0]).name) @@ -604,6 +655,8 @@ def test_a_readable_tree_builds_normally(tmp_path: Path, monkeypatch: pytest.Mon (tmp_path / "a.php").write_text(" None: + """`-J-Xmx` sizes only the launcher; the JVM that runs the script is forked without it, so the heap has + to travel as JAVA_TOOL_OPTIONS too -- appended, so an operator's own options survive.""" + from openultrasast.cpg.backend import JoernBackend + + monkeypatch.setenv("JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8") + monkeypatch.setenv("OPENULTRASAST_CPG_HEAP_MB", "1536") + backend = JoernBackend(runner=lambda c, **k: None) + assert backend._heap_flag() == "-J-Xmx1536m" + assert backend._jvm_env()["JAVA_TOOL_OPTIONS"] == "-Dfile.encoding=UTF-8 -Xmx1536m" + monkeypatch.delenv("JAVA_TOOL_OPTIONS") + assert JoernBackend(runner=lambda c, **k: None, heap_mb=1024)._jvm_env()["JAVA_TOOL_OPTIONS"] == "-Xmx1024m" + + +def test_a_frontend_build_gets_its_overlays_once_at_build_time(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """php2cpg writes a raw graph; `importCpg` computes every missing overlay each time a JVM opens it. + + That cost 54 s per query batch on a 637-file plugin and ran WP Statistics out of heap. The build runs + `queries/overlay.sc` once, after the last frontend attempt, and the saved graph replaces the raw one + under the same path -- and a failed overlay pass keeps the raw graph rather than losing it. + """ + import subprocess + + from openultrasast.cpg.backend import JoernBackend + + commands: list[list[str]] = [] + + def runner(command, **kwargs): # type: ignore[no-untyped-def] + commands.append(list(command)) + if "-r" in command: + return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) + if "--script" in command: + return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") + Path(command[command.index("-o") + 1]).write_text("raw") + return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") + monkeypatch.setattr("shutil.which", lambda name: f"/opt/joern/{name}") + + result = JoernBackend(runner=runner).build(tmp_path, language="php") + assert result is not None + overlay_runs = [c for c in commands if _is_overlay(c)] + assert len(overlay_runs) == 1, "once, not per attempt" + assert f"cpgFile={result.cpg_path}" in overlay_runs[0] + assert result.cpg_path.read_text() == "cpg+overlays", "the overlaid graph replaced the raw one, same path" + assert not (result.cpg_path.parent / "workspace").exists(), "Joern's workspace does not outlive the step" + + def failing(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="OutOfMemoryError") + return runner(command, **kwargs) + + kept = JoernBackend(runner=failing).build(tmp_path, language="php") + assert kept is not None and kept.cpg_path.read_text() == "raw", "a failed overlay pass degrades, it does not lose the graph" + + +def test_a_source_file_the_size_of_a_data_table_is_excluded_and_named(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """WP Statistics vendors two browser-profile files of 1.6 MB and 1.5 MB, each one array literal. They + were 60% of its graph, and the reaching-definitions overlay never finished on them -- 25 minutes at 4 GB, + OutOfMemoryError at every `--max-num-def`, because one giant assignment is ONE definition. Nothing a + scan could name flows through a lookup table, so a file over `MAX_SOURCE_BYTES` stays out of the build, + is excluded by name, and is reported rather than silently dropped. + """ + import subprocess + + from openultrasast.cpg.backend import MAX_SOURCE_BYTES, JoernBackend + + (tmp_path / "code.php").write_text(" 1,\n" * (MAX_SOURCE_BYTES // 10) + "];\n") + commands: list[list[str]] = [] + + def runner(command, **kwargs): # type: ignore[no-untyped-def] + if _is_overlay(command): + return _overlaid(command, kwargs.get("cwd")) + if "-r" in command: + return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") + commands.append(list(command)) + if "--script" in command: + body = '---OUSAST-CPG-BEGIN---\n{"files": "1", "methods": "1"}\n---OUSAST-CPG-END---\n' + return subprocess.CompletedProcess(args=command, returncode=0, stdout=body, stderr="") + Path(command[command.index("-o") + 1]).write_text("cpg") + return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on") + monkeypatch.setattr("shutil.which", lambda name: f"/opt/bin/{name}") + + result = JoernBackend(runner=runner).build(tmp_path, language="php") + assert result is not None + build = next(c for c in commands if "-o" in c) + assert "--exclude" in build and "vendor/profiles.php" in build, "the data table is excluded from the build by name" + assert not any("code.php" in part for part in build), "and the code is not" + assert [Path(name).name for name in result.unparsed] == ["profiles.php"], "and it is reported, not silently dropped" diff --git a/tests/test_cpg_queries_compile.py b/tests/test_cpg_queries_compile.py index f00f3cd..9b5646d 100644 --- a/tests/test_cpg_queries_compile.py +++ b/tests/test_cpg_queries_compile.py @@ -26,7 +26,7 @@ def _joern() -> str | None: @pytest.mark.skipif(_joern() is None, reason="joern is not installed on this machine") -@pytest.mark.parametrize("name", ["taint", "dominance", "config", "census"]) +@pytest.mark.parametrize("name", ["taint", "dominance", "config", "census", "overlay"]) def test_each_query_compiles_and_answers(name: str, tmp_path: Path) -> None: """Compile and RUN each query against a graph, and require a fenced payload back. @@ -49,11 +49,11 @@ def test_each_query_compiles_and_answers(name: str, tmp_path: Path) -> None: pytest.skip(f"could not build a sample cpg: {(built.stderr or '')[-200:]}") command = [_joern() or "joern", "--script", str(script.resolve()), "--param", f"cpgFile={cpg}"] - if name != "census": + if name not in {"census", "overlay"}: requests = tmp_path / "requests.json" requests.write_text(json.dumps({"0": {"sources": "$_GET", "sinks": "$wpdb->query", "function": "handler"}})) command += ["--param", f"requestsFile={requests}"] - done = subprocess.run(command, capture_output=True, text=True, timeout=900, check=False) + done = subprocess.run(command, capture_output=True, text=True, timeout=900, check=False, cwd=str(tmp_path)) assert done.returncode == 0, f"{name}.sc did not run: {(done.stderr or done.stdout or '')[-500:]}" assert extract_payload(done.stdout or "") is not None, f"{name}.sc produced no parseable payload"