From b6cccd1444bfab9aad0dc8fb7725521bf078acaa Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:30:12 +0200 Subject: [PATCH] fix(bench,cpg): the position benchmark builds in the recipe's language and reports per repository; a doubly-failed build disposes its scratch WP Statistics's first region is JavaScript, and a build asked for as JavaScript went looking for jssrc2cpg, failed both launchers, lost the whole run's output and left a 9.6 MB scratch directory behind. Also notes upstream: joern#6281 is fixed in v4.0.625. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .kiro/specs/contributor-scan/tasks.md | 6 ++++++ benchmarks/ranking/position.py | 15 ++++++++++----- src/openultrasast/cpg/backend.py | 1 + tests/test_cpg_backend.py | 24 ++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index f2b4280..13e9c03 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -920,6 +920,12 @@ findings across 40 files. First completion of this scan on this repository. The 2,400 s default is still too low for it and stays -- a per-push scan of 4,463 regions is the flow-aware-ranking spec's problem, and 250 findings is the number that spec now has to concentrate. + - **Upstream, 2026-09-11:** joernio/joern#6281 (the `global`-inside-a-closure miscompile) is closed as + fixed -- same root cause as #6269, fixed by #6270, released in **v4.0.625** (we run 4.0.623). The + maintainer verified our reproducer on master: no node with two AST parents, overlays apply cleanly. + So `_files_with_frontend_defect` and its exclusion are a workaround for two releases and should become + version-gated: verify on 4.0.625 (the reproducer, then pmpro and WP Statistics with NO exclusions), and + check whether #6283 (`--overlaysonly`) is also gone there. - **Open, and the maintainer's call, not mine:** `includes/vendor/` is 207 of WP Statistics's 357 PHP files and is not excluded by `preprocess.IGNORED_DIRS`, so vendored third-party code is both scanned and in the graph. Whether a project's vendored code is in scope is policy; the size guard above is diff --git a/benchmarks/ranking/position.py b/benchmarks/ranking/position.py index 36bcb1c..243bfb0 100644 --- a/benchmarks/ranking/position.py +++ b/benchmarks/ranking/position.py @@ -38,7 +38,7 @@ def ordered_regions(root: Path) -> list[ScanRegion]: return sorted(regions, key=lambda r: (not r.shipped, -r.rank, r.path, r.function or "")) -def evidence_ordered_regions(root: Path, regions: list[ScanRegion]) -> tuple[list[ScanRegion], dict[str, object]]: +def evidence_ordered_regions(root: Path, regions: list[ScanRegion], language: str) -> tuple[list[ScanRegion], dict[str, object]]: """The regions in the order phase 2 spends the budget: by each region's best (tier, score), from a real evidence pass over EVERY region. Needs Joern. The ordering function is the scan's own, imported.""" import time @@ -48,7 +48,9 @@ def evidence_ordered_regions(root: Path, regions: list[ScanRegion]) -> tuple[lis backend = resolve_cpg_backend() started = time.monotonic() - cpg = backend.build(root, language=regions[0].language if regions else "") + # The RECIPE's language, never the first region's: WP Statistics's first region is JavaScript, and a + # build asked for as JavaScript went looking for jssrc2cpg and lost the whole repository's measurement. + cpg = backend.build(root, language=language) build_seconds = round(time.monotonic() - started, 1) if cpg is None: raise SystemExit(f"no graph for {root}: {getattr(backend, 'last_failure', '')}") @@ -70,11 +72,11 @@ def evidence_ordered_regions(root: Path, regions: list[ScanRegion]) -> tuple[lis return order_by_evidence(regions, evidence), facts -def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidence: bool = False) -> dict[str, object]: +def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidence: bool = False, language: str = "") -> dict[str, object]: regions = ordered_regions(root) facts: dict[str, object] = {} if by_evidence: - regions, facts = evidence_ordered_regions(root, regions) + regions, facts = evidence_ordered_regions(root, regions, language) tiers = Counter(r.rank for r in regions) largest = max(tiers.values()) if tiers else 0 rows: list[dict[str, object]] = [] @@ -136,7 +138,10 @@ def main() -> int: {"id": k.id, "family": k.family, "file": k.file, "function": k.function, "line": k.line, "in_scope": k.in_scope} for k in recipe.known ] - results.append(measure(recipe.name, root, known, by_evidence=args.evidence)) + result = measure(recipe.name, root, known, by_evidence=args.evidence, language=recipe.language) + results.append(result) + # One line per repository as it lands, so a failure on the next repository cannot lose this one. + print(json.dumps(result), file=sys.stderr, flush=True) if args.json: print(json.dumps(results, indent=2)) diff --git a/src/openultrasast/cpg/backend.py b/src/openultrasast/cpg/backend.py index f46e41b..5521a93 100644 --- a/src/openultrasast/cpg/backend.py +++ b/src/openultrasast/cpg/backend.py @@ -305,6 +305,7 @@ class JoernBackend: logger.warning("cpg build failed for %s: %s", root, detail) retried = self._build_with_frontend(root, cpg_path, scratch, language) if retried is None: + dispose() # a failed build's scratch is nobody's to sweep six hours later return None self._apply_overlays(cpg_path, scratch) unparsed = retried diff --git a/tests/test_cpg_backend.py b/tests/test_cpg_backend.py index 269b9f8..497df83 100644 --- a/tests/test_cpg_backend.py +++ b/tests/test_cpg_backend.py @@ -1018,3 +1018,27 @@ def test_the_census_checks_the_instrument(caplog: pytest.LogCaptureFixture, monk )._graph_census(tmp_path / "c.bin", tmp_path, "php") assert "512 MB heap where 2048 MB is configured" in caplog.text assert "recompute" not in caplog.text + + +def test_a_build_that_fails_on_both_launchers_leaves_no_scratch_behind(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Measured: a measurement that asked for the wrong language failed both `joern-parse` and the frontend + and left a 9.6 MB `ousast-cpg-*` directory in /tmp. Failure paths dispose too.""" + import subprocess + import tempfile + + from openultrasast.cpg.backend import JoernBackend + + (tmp_path / "a.php").write_text("