From 3b3b96be5f668574005464b42b6471ad74164175 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 11 Sep 2026 13:52:17 +0200 Subject: [PATCH] fix(cpg): the interpreter precondition probes the frontend's own parser script; 4.0.625 verified php2cpg drives PHP-Parser through `php` from a script inside its own install. A second install the containerised interpreter had no mount for parsed 1 of 637 files and exited 0 -- caught by the build census, and now refused by name before any build. On 4.0.625 with the closure exclusion gated off: pmpro 637 of 637 in 100 s, WP Statistics 115 s with only the two data tables excluded. Phase-2 v5 verdict and the within-tier reading recorded in the brief. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .kiro/specs/contributor-scan/tasks.md | 8 ++++++- .kiro/specs/flow-aware-ranking/brief.md | 15 +++++++++++++ src/openultrasast/cpg/backend.py | 19 ++++++++++++++++ tests/test_cpg_backend.py | 29 +++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index 13e9c03..69a66f3 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -925,7 +925,13 @@ maintainer verified our reproducer on master: no node with two AST parents, overlays apply cleanly. So `_files_with_frontend_defect` and its exclusion are a workaround for two releases and should become version-gated: verify on 4.0.625 (the reproducer, then pmpro and WP Statistics with NO exclusions), and - check whether #6283 (`--overlaysonly`) is also gone there. + check whether #6283 (`--overlaysonly`) is also gone there. **Done, 2026-09-11:** the reproducer builds + and loads cleanly on 4.0.625; #6283 still reproduces there (commented upstream). Through the backend, + with the exclusion version-gated off: **pmpro 637 of 637 files in 100 s, WP Statistics 115 s with only + the two data tables excluded by name.** One more instrument finding on the way: the second install was + not mounted into the containerised `php`, so php2cpg 4.0.625 parsed 1 of 637 files and exited 0 -- + caught by the build census, and the interpreter precondition now probes the frontend's own parser + script as well as the repository. - **Open, and the maintainer's call, not mine:** `includes/vendor/` is 207 of WP Statistics's 357 PHP files and is not excluded by `preprocess.IGNORED_DIRS`, so vendored third-party code is both scanned and in the graph. Whether a project's vendored code is in scope is policy; the size guard above is diff --git a/.kiro/specs/flow-aware-ranking/brief.md b/.kiro/specs/flow-aware-ranking/brief.md index fb80f5a..3d75c1d 100644 --- a/.kiro/specs/flow-aware-ranking/brief.md +++ b/.kiro/specs/flow-aware-ranking/brief.md @@ -190,6 +190,21 @@ A phase that cannot show that is not an improvement, it is a story. precisely the parameter-fed fields a sink-bearing region reads -- memoised per (family, file, field) for the batch, which the arbiter's memos were not until today -- and for nothing else. + **v5, measured, and the phase-2 verdict on it** (`2026-09-11-evidence-rank-v5.json`, with the head of + each order recorded): pmpro **69**, wpstatistics **172**, mwwpform **68**, vampi 16. The flow confirms + every parameter-fed field -- a constructor's `$this->x = $x` trivially flows from its parameter -- so + v5 equals v3 for mwwpform; the v2 "30" came from the older scoping (the region's file, not the + field's), which happened to fit that one case. 68 is the honest structural number for that shape. + + Reading the heads: every region ahead of a CVE is in the CVE's own tier on a tied or higher score -- + pmpro's CVE scores 7.0 with 23 regions at 10.0, 13 at 8.5, 10 at 8.0 and 14 at 7.0 ahead of it; + wpstatistics's sits behind 96 regions tied at 7.0. The tiers are done; the ties are the whole of what + is left, exactly as the cross-assessment said. One exact fact was missing from the vector: `shipped`, + the build's own word on what is product, was on the region and not in the score, and 10 of the 67 + regions ahead of MW WP Form's CVE were `test-mail.php`. Weight −2.0 for an unshipped file (v6, in + flight). After that, within-tier order is phase 3's -- a model reading the code the ties cannot -- + or one more exact path fact (`distance`), and the gate stays unmet until one of them moves it. + - **Phase 3 — the LLM ranker.** Same evidence, ordered by a model. Gate: beats phase 2 leave-one-repository-out. If it does not, phase 2 ships and phase 3 does not. - **Phase 4 — break the closed loop.** An exploration slice (~10% of budget sampled outside the top-K) and diff --git a/src/openultrasast/cpg/backend.py b/src/openultrasast/cpg/backend.py index 92f67b3..1c99491 100644 --- a/src/openultrasast/cpg/backend.py +++ b/src/openultrasast/cpg/backend.py @@ -87,6 +87,11 @@ _FRONTENDS = { _PREFER_FRONTEND = frozenset({"php"}) # The extensions that count as source for a language, used to ask whether a graph is missing anything. _LANGUAGE_EXTENSIONS: dict[str, tuple[str, ...]] = {"php": (".php",)} +# The script a frontend drives THROUGH the interpreter, relative to the frontend binary's directory. The +# interpreter has to be able to read it as much as the repository: php2cpg 4.0.625 in an install the +# containerised `php` had no mount for parsed 1 of 637 files -- "Could not open input file" 64 times, +# exit 0 -- and read for a minute as a frontend regression. +_FRONTEND_PARSER: dict[str, str] = {"php": "frontends/php2cpg/bin/php-parser/php-parser.php"} # A source file above this size is a data table, not code: WP Statistics carries two vendored browser # profile files of 1.6 MB and 1.5 MB, each one array literal, and they were 60% of the graph (9.6 MB with @@ -429,6 +434,20 @@ class JoernBackend: f"write an empty graph. Check that {interpreter} is a real interpreter with access to this " f"tree -- a container or wrapper without the right mount fails exactly this way." ) + # The same question about the frontend's own parser script, which the interpreter also has to open. + frontend = _FRONTENDS.get(language.lower()) + relative = _FRONTEND_PARSER.get(language.lower()) + frontend_binary = shutil.which(frontend) if frontend else None + if frontend_binary and relative: + parser = Path(frontend_binary).resolve().parent / relative + if parser.is_file(): + completed = self._run([binary, "-r", probe, str(parser)], timeout=60) + if completed is not None and completed.returncode != 0: + return ( + f"the {interpreter} interpreter cannot read {parser}, the script {frontend} drives it with, " + f"so every file would fail to parse and the graph would be empty. A container or wrapper " + f"that mounts the repository but not this install fails exactly this way." + ) return "" def _build_sharded(self, root: Path, scratch: Path, language: str) -> tuple[tuple[Path, ...], tuple[str, ...]]: diff --git a/tests/test_cpg_backend.py b/tests/test_cpg_backend.py index c7e5d13..78b9234 100644 --- a/tests/test_cpg_backend.py +++ b/tests/test_cpg_backend.py @@ -1073,3 +1073,32 @@ def test_the_closure_exclusion_stops_at_the_release_that_fixed_it(tmp_path: Path monkeypatch.setattr("shutil.which", lambda name: "/opt/bin/joern") assert joern_version() is None assert [Path(n).name for n in backend._files_with_frontend_defect(tmp_path, "php")] == ["bad.php"], "unknown keeps the workaround" + + +def test_the_interpreter_must_read_the_frontend_parser_too(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """php2cpg drives PHP-Parser through `php` from a script inside its own install. An interpreter that can + read the repository but not that script parses nothing, exits 0, and looks like a frontend regression -- + measured with a second Joern install the containerised `php` had no mount for. Refused by name.""" + import subprocess + + from openultrasast.cpg.backend import JoernBackend + + (tmp_path / "repo").mkdir() + (tmp_path / "repo" / "a.php").write_text("