perf(cpg): build JavaScript in one engine start, gated on the census (M2 11.6)

JavaScript was the only language paying two engine starts: the declared frontend
and then a separate overlay pass. joern-parse applies the overlays itself. On the
pinned Node tree that is 11.0 s against 21.0 s.

The declared frontend stays as the fallback, because it is what the versioned
retention adaptation and the explicit exclusion policy run through, and the census
decides per build: a graph that does not hold every readable declared file is
rebuilt through the frontend rather than trusted. Exclusions keep the frontend path
outright.

Measured under the production retention environment, both paths produce the same 44
files, the same 579 methods, the same file names, the same five overlays, the
Gruntfile and all eighteen first-party test files, with no file present in one
graph and missing from the other.

My first measurement did not establish that. It ran without the retention
environment, both paths dropped to 25 files, and the apparent equivalence was two
runs discarding the same things. The existing control named for that decision
failed against the change built on it, which is exactly what it is for; the
evidence was wrong, not the control. That control now asserts the intent directly:
the faster path is tried first, and a short census rebuilds through the frontend.

1,401 tests passed, nine skipped. A build saving is not the latency gate; the
profile rerun decides that. Rollout stays NO-GO.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

benchmarks/measurements/2026-09-17-javascript-build-path.json

@@ -0,0 +1,150 @@
+{
+  "experiment": "javascript-build-path-v1",
+  "root": "/cache/nodegoat/c5cb68a7084e",
+  "paths": [
+    {
+      "path": "frontend_then_overlay",
+      "frontend_seconds": 6.20875704404898,
+      "overlay_seconds": 14.808171501965262,
+      "total_seconds": 21.016928546014242,
+      "frontend_exit": 0,
+      "overlay_exit": 0,
+      "census": {
+        "files": "44",
+        "methods": "579",
+        "calls": "4134",
+        "file_names": [
+          "Gruntfile.js",
+          "app/assets/js/chart/chart-data-morris.js",
+          "app/assets/js/tour/redirects-steps.js",
+          "app/data/allocations-dao.js",
+          "app/data/benefits-dao.js",
+          "app/data/contributions-dao.js",
+          "app/data/memos-dao.js",
+          "app/data/profile-dao.js",
+          "app/data/research-dao.js",
+          "app/data/user-dao.js",
+          "app/routes/allocations.js",
+          "app/routes/benefits.js",
+          "app/routes/contributions.js",
+          "app/routes/error.js",
+          "app/routes/index.js",
+          "app/routes/memos.js",
+          "app/routes/profile.js",
+          "app/routes/research.js",
+          "app/routes/session.js",
+          "app/routes/tutorial.js",
+          "artifacts/db-reset.js",
+          "config/config.js",
+          "config/env/all.js",
+          "config/env/development.js",
+          "config/env/production.js",
+          "config/env/test.js",
+          "server.js",
+          "test/e2e/integration/allocations_spec.js",
+          "test/e2e/integration/benefits_spec.js",
+          "test/e2e/integration/contributions_spec.js",
+          "test/e2e/integration/dashboard_spec.js",
+          "test/e2e/integration/general_spec.js",
+          "test/e2e/integration/learn_spec.js",
+          "test/e2e/integration/login_spec.js",
+          "test/e2e/integration/logout_spec.js",
+          "test/e2e/integration/memos_spec.js",
+          "test/e2e/integration/profile_spec.js",
+          "test/e2e/integration/research_spec.js",
+          "test/e2e/integration/signup_spec.js",
+          "test/e2e/integration/tutorial_spec.js",
+          "test/e2e/plugins/index.js",
+          "test/e2e/support/commands.js",
+          "test/e2e/support/index.js",
+          "test/security/profile-test.js"
+        ],
+        "overlays": "base,controlflow,typerel,callgraph,dataflowOss",
+        "maxHeapMB": "2048"
+      }
+    },
+    {
+      "path": "joern_parse_only",
+      "total_seconds": 10.995579135953449,
+      "exit": 0,
+      "stderr_tail": "Picked up JAVA_TOOL_OPTIONS: -Xmx2048m\n2026-09-17 06:41:16.285 WARN  CfgCreator$               Using order fallback for try statement finally body\n",
+      "census": {
+        "files": "44",
+        "methods": "579",
+        "calls": "4134",
+        "file_names": [
+          "Gruntfile.js",
+          "app/assets/js/chart/chart-data-morris.js",
+          "app/assets/js/tour/redirects-steps.js",
+          "app/data/allocations-dao.js",
+          "app/data/benefits-dao.js",
+          "app/data/contributions-dao.js",
+          "app/data/memos-dao.js",
+          "app/data/profile-dao.js",
+          "app/data/research-dao.js",
+          "app/data/user-dao.js",
+          "app/routes/allocations.js",
+          "app/routes/benefits.js",
+          "app/routes/contributions.js",
+          "app/routes/error.js",
+          "app/routes/index.js",
+          "app/routes/memos.js",
+          "app/routes/profile.js",
+          "app/routes/research.js",
+          "app/routes/session.js",
+          "app/routes/tutorial.js",
+          "artifacts/db-reset.js",
+          "config/config.js",
+          "config/env/all.js",
+          "config/env/development.js",
+          "config/env/production.js",
+          "config/env/test.js",
+          "server.js",
+          "test/e2e/integration/allocations_spec.js",
+          "test/e2e/integration/benefits_spec.js",
+          "test/e2e/integration/contributions_spec.js",
+          "test/e2e/integration/dashboard_spec.js",
+          "test/e2e/integration/general_spec.js",
+          "test/e2e/integration/learn_spec.js",
+          "test/e2e/integration/login_spec.js",
+          "test/e2e/integration/logout_spec.js",
+          "test/e2e/integration/memos_spec.js",
+          "test/e2e/integration/profile_spec.js",
+          "test/e2e/integration/research_spec.js",
+          "test/e2e/integration/signup_spec.js",
+          "test/e2e/integration/tutorial_spec.js",
+          "test/e2e/plugins/index.js",
+          "test/e2e/support/commands.js",
+          "test/e2e/support/index.js",
+          "test/security/profile-test.js"
+        ],
+        "overlays": "base,controlflow,typerel,callgraph,dataflowOss",
+        "maxHeapMB": "2048"
+      }
+    }
+  ],
+  "source_files": 50,
+  "source_bytes": 367111,
+  "retention": {
+    "frontend_has_gruntfile": true,
+    "joern_parse_has_gruntfile": true,
+    "frontend_test_files": 18,
+    "joern_parse_test_files": 18,
+    "only_in_frontend": [],
+    "only_in_joern_parse": []
+  },
+  "comparison": {
+    "same_files": true,
+    "same_methods": true,
+    "same_overlays": true,
+    "same_file_names": true,
+    "seconds_saved": 10.02
+  },
+  "milestone": "M2 task 11.6",
+  "instrument_note": "A first run without OUSAST_INCLUDE_TESTS and OUSAST_INCLUDE_BUILD_CONFIGS reported 25 files from both paths and appeared to show equivalence. It did not: both runs dropped the retained files, so the comparison said nothing about the retention adaptation the frontend path exists for. An existing control caught the change built on it. This run sets the production environment.",
+  "decision": "JavaScript now builds through joern-parse, one engine start including overlays, and falls back to the declared frontend plus overlay pass whenever the census is short. Both paths hold the same 44 files, 579 methods, file names, overlays, the Gruntfile and all eighteen first-party test files.",
+  "limits": [
+    "One pinned JavaScript tree. PHP keeps the frontend path, which it needs for exclusions and retries.",
+    "A build saving is not the latency gate: the profile rerun decides that."
+  ]
+}

src/openultrasast/cpg/backend.py

@@ -552,8 +552,30 @@ class JoernBackend:
                     execution_diagnostics=lambda: tuple(self._diagnostics),
                 )
         if language.lower() in {"javascript", "typescript"}:
-            # Use the declared frontend rather than language autodetection. The versioned
+            # M2 11.6: `joern-parse` applies the overlays itself, so it is ONE engine start where the
-            # retention adaptation consumes the explicit include_tests subprocess policy.
+            # frontend plus a separate overlay pass is two. On the pinned Node tree that is 11.0 s
+            # against 21.0 s, and JavaScript was the only language paying twice.
+            #
+            # The declared frontend stays, because it is what the versioned retention adaptation and
+            # the explicit exclusion policy run through, and a build nine seconds faster that quietly
+            # drops a file is worse rather than better. Measured under the production retention
+            # environment both paths hold the same 44 files and 579 methods, the same file names, the
+            # same overlays, the Gruntfile and all eighteen first-party test files. That is why this
+            # is allowed at all, and the census still decides per build: an incomplete graph is
+            # rebuilt through the frontend rather than trusted.
+            parsed_unparsed: tuple[str, ...] | None = None
+            if not exclude and parse is not None:
+                command = [parse, self._heap_flag(), str(root), "--output", str(cpg_path)]
+                completed = self._run(command, timeout=self.build_timeout, cwd=scratch)
+                if completed is not None and completed.returncode == 0 and cpg_path.is_file():
+                    if self._graph_is_complete(cpg_path, root, language):
+                        parsed_unparsed = _unparsed_files(completed)
+                    else:
+                        self._note("joern_parse_census_incomplete")
+                        logger.info("joern-parse produced an incomplete graph for %s; rebuilding through the frontend", root)
+            if parsed_unparsed is not None:
+                unparsed = parsed_unparsed
+            else:
                 direct = self._build_with_frontend(root, cpg_path, scratch, language, exclude)
                 if direct is None:
                     dispose()

tests/test_cpg_backend.py

@@ -1135,27 +1135,41 @@ def test_the_interpreter_must_read_the_frontend_parser_too(tmp_path: Path, monke
     assert "php-parser.php" in backend.last_failure and "mounts the repository but not this install" in backend.last_failure
 
 
-def test_javascript_partition_uses_declared_frontend_not_legacy_autodetection(tmp_path, monkeypatch):
+def test_javascript_falls_back_to_the_declared_frontend_when_the_census_is_short(tmp_path, monkeypatch):
+    """M2 11.6: one engine start is allowed only while the graph still holds every declared file.
+
+    The retention adaptation runs through the declared frontend, and it is what took the Node census
+    from 43 files to 44. So an incomplete graph from the faster path is rebuilt rather than trusted.
+    """
     from openultrasast.cpg.backend import JoernBackend
 
     (tmp_path / "tests").mkdir()
     (tmp_path / "tests/probe.js").write_bytes(b"function first_party_test(x) { return x; }")
+    (tmp_path / "app.js").write_bytes(b"function handler(x) { return x; }")
     commands = []
 
     def runner(command, **kwargs):
         if _is_overlay(command):
             return _overlaid(command, kwargs.get("cwd"))
-        commands.append(command)
+        commands.append(Path(command[0]).name)
-        assert Path(command[0]).name == "jssrc2cpg"
+        if command[0].endswith("census.sc") or "census" in " ".join(command):
-        Path(command[command.index("-o") + 1]).write_bytes(b"graph")
+            # One file short of the two on disk: the faster path must not be trusted.
+            body = '{"files":"1","methods":"1","overlays":"base,dataflowOss","maxHeapMB":"2048"}'
+            payload = "---OUSAST-CPG-BEGIN---\n" + body + "\n---OUSAST-CPG-END---"
+            return subprocess.CompletedProcess(command, 0, payload, "")
+        target = command[command.index("-o") + 1] if "-o" in command else command[command.index("--output") + 1]
+        Path(target).write_bytes(b"graph")
         return subprocess.CompletedProcess(command, 0, "", "")
 
     monkeypatch.setattr("shutil.which", lambda name: f"/opt/joern/{name}")
     monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on")
-    result = JoernBackend(runner=runner).build(tmp_path, language="javascript")
+    backend = JoernBackend(runner=runner)
+    result = backend.build(tmp_path, language="javascript")
     try:
         assert result is not None
-        assert len(commands) == 1
+        assert "joern-parse" in commands, "the one-start path should be tried first"
+        assert "jssrc2cpg" in commands, "a short census must rebuild through the declared frontend"
+        assert "joern_parse_census_incomplete" in backend._diagnostics
     finally:
         if result is not None:
             result.cleanup()