feat(contributor-scan): retain first-party JavaScript tests in graphs

ops/frontend-retention/README.md

@@ -0,0 +1,36 @@
+# First-party JavaScript test retention
+
+This source adaptation implements contributor-scan task 2.18 for the pinned Linux
+x86-64 Joern 4.0.625 installation. ASTGen 3.50.1 excludes test directories and
+filenames; jssrc2cpg additionally excludes test suffixes. Both filters must accept
+the explicit inclusion policy for a retained source file to reach the graph.
+
+`install.py` verifies pinned upstream source and tool archives and the original
+jssrc2cpg jar, rebuilds ASTGen with its frozen dependency lockfile, and compiles
+the adapted Scala source against Joern's shipped compiler and libraries. The
+changes affect test filtering only. Original input paths, bytes and imports are
+preserved. Vendor and user exclusions still apply.
+
+Docker and `ops/joern.py` run the same installer. A manual installation requires
+Python 3.12 or later, Java 21, network access for the pinned build inputs, and a
+pristine matching Joern installation:
+
+```bash
+python3 ops/frontend-retention/install.py /opt/joern-cli
+```
+
+The installer writes `ousast-frontend-retention-v1/manifest.json` under the Joern
+root, including builder, dependency and output hashes. A repeated invocation
+verifies the installed outputs and needs no network. If the recipe or installed
+outputs change, reinstall the pristine pinned engine before rebuilding; the
+installer refuses to layer an unverified adaptation onto modified jars.
+
+The adapted frontend keeps stock behavior unless `OUSAST_INCLUDE_TESTS=1`.
+`JoernBackend` explicitly enables it by default; `include_tests=False` supplies
+the regression control. Direct frontend users must set the variable themselves.
+Engine provenance must include the installed adaptation and effective policy:
+the upstream version string alone cannot distinguish these graphs.
+
+After building the image, run `ops/smoke_partitions.py` as described in `ops/README.md`.
+Its off control uses `OUSAST_SMOKE_INCLUDE_TESTS=0` and must fail the test-retention
+assertion. First-party test retention does not imply security capability admission.

ops/frontend-retention/install.py

@@ -0,0 +1,151 @@
+"""Build the approved test-retention adaptation from checksum-pinned upstream sources.
+
+Usage: python3 install.py /opt/joern-cli. Requires curl, unzip, Java 21 and Python3.
+Only the two upstream input filters change. OUSAST_INCLUDE_TESTS=1 opts in;
+unset/0 keeps upstream behavior. Sources, lockfile and build identity are retained.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import urllib.request
+import zipfile
+from pathlib import Path
+
+PINS = {
+    "astgen.tar.gz": (
+        "https://codeload.github.com/joernio/astgen-monorepo/tar.gz/refs/tags/javascript-astgen/v3.50.1",
+        "8d9728dca8eab694a0f07bcd7a1c9a88368cb7bd354fc19c2ee8ca8611ff869a",
+    ),
+    "AstGenRunner.scala": (
+        "https://raw.githubusercontent.com/joernio/joern/v4.0.625/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala",
+        "9963f949360d7e17baed1bd8875b717b2855745cf7260eef8a87d17533a54268",
+    ),
+    "bun.zip": (
+        "https://github.com/oven-sh/bun/releases/download/bun-v1.4.2/bun-linux-x64.zip",
+        "36368faef7527875d5ffa52e53cd48021741f2a83eb6208a8dd64068d422a913",
+    ),
+}
+JAR_SHA256 = "49fe488a79cc52b4e1d2e056143ec414f59e0007ed25f054c5fa7ae4aa34dff3"
+
+
+def digest(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def replace_once(path: Path, before: str, after: str) -> None:
+    source = path.read_text()
+    if source.count(before) != 1:
+        raise ValueError(f"Unexpected source shape: {path.name}")
+    path.write_text(source.replace(before, after))
+
+
+def build(root: Path) -> None:
+    jar = root / "frontends/jssrc2cpg/lib/io.joern.jssrc2cpg-4.0.625.jar"
+    installed = root / "ousast-frontend-retention-v1/manifest.json"
+    if installed.is_file():
+        previous = json.loads(installed.read_text())
+        if (
+            previous.get("builder_sha256") == digest(Path(__file__))
+            and previous.get("adapted_jar_sha256") == digest(jar)
+            and previous.get("astgen_sha256") == digest(root / "frontends/jssrc2cpg/bin/astgen/astgen-linux")
+        ):
+            print("Verified installed frontend-retention adaptation")
+            return
+    if digest(jar) != JAR_SHA256:
+        raise ValueError("Expected pristine Joern 4.0.625 jssrc2cpg jar")
+    with tempfile.TemporaryDirectory(prefix="ousast-frontend-build-") as temporary:
+        work = Path(temporary)
+        for name, (url, expected) in PINS.items():
+            with urllib.request.urlopen(url) as response:
+                (work / name).write_bytes(response.read())
+            if digest(work / name) != expected:
+                raise ValueError(f"Checksum mismatch: {name}")
+        with tarfile.open(work / "astgen.tar.gz") as archive:
+            archive.extractall(work, filter="data")
+        with zipfile.ZipFile(work / "bun.zip") as archive:
+            archive.extractall(work)
+        bun = work / "bun-linux-x64/bun"
+        bun.chmod(0o755)
+        astgen = work / "astgen-monorepo-javascript-astgen-v3.50.1/javascript-astgen"
+        file_utils = astgen / "src/FileUtils.ts"
+        replace_once(
+            file_utils,
+            "const IGNORE_DIRS_SET = new Set(Defaults.IGNORE_DIRS.map((d) => d.toLowerCase()))",
+            """const INCLUDE_TESTS = process.env.OUSAST_INCLUDE_TESTS === "1"
+const TEST_DIRS = new Set(["test", "tests", "e2e", "e2e-beta", "cypress", "__tests__", "__mocks__"])
+const IGNORE_DIRS_SET = new Set(Defaults.IGNORE_DIRS.filter(d => !INCLUDE_TESTS || !TEST_DIRS.has(d)).map(d => d.toLowerCase()))
+const IGNORE_FILE_PATTERN = INCLUDE_TESTS
+    ? new RegExp("(chunk-vendors|app~|conf|[.-]min|\\\\.d)\\\\.(js|jsx|cjs|mjs|xsjs|xsjslib|ts|tsx)$", "i")
+    : Defaults.IGNORE_FILE_PATTERN""",
+        )
+        replace_once(
+            file_utils,
+            'dirName.startsWith("__") ||',
+            '(dirName.startsWith("__") && !(INCLUDE_TESTS && TEST_DIRS.has(dirName.toLowerCase()))) ||',
+        )
+        replace_once(file_utils, "Defaults.IGNORE_FILE_PATTERN.test(fileName)", "IGNORE_FILE_PATTERN.test(fileName)")
+        scala = work / "AstGenRunner.scala"
+        replace_once(
+            scala,
+            "lazy val isIgnoredTest = IgnoredTestsRegex.exists(_.matches(filePath))",
+            'lazy val isIgnoredTest = !sys.env.get("OUSAST_INCLUDE_TESTS").contains("1") && IgnoredTestsRegex.exists(_.matches(filePath))',
+        )
+        # Upstream stages embedded TS standard-library assets; retain that build recipe,
+        # selecting just the supported Linux x64 target instead of all release platforms.
+        script = astgen / "scripts/build-binaries.ts"
+        source = script.read_text()
+        start = source.index("const targets = [")
+        end = source.index("] as const", start) + len("] as const")
+        script.write_text(
+            source[:start] + 'const targets = [{target: "bun-linux-x64", outfile: "./astgen-linux-x64"}] as const' + source[end:]
+        )
+        subprocess.run([str(bun), "install", "--frozen-lockfile"], cwd=astgen, check=True)
+        subprocess.run([str(bun), "scripts/sync-version.ts"], cwd=astgen, check=True)
+        subprocess.run([str(bun), "scripts/build-binaries.ts"], cwd=astgen, check=True)
+        classes = work / "classes"
+        classes.mkdir()
+        jars = sorted(set(root.glob("lib/*.jar")) | set(root.glob("frontends/jssrc2cpg/lib/*.jar")))
+        classpath = ":".join(map(str, jars))
+        subprocess.run(
+            ["java", "-cp", classpath, "dotty.tools.dotc.Main", "-classpath", classpath, "-d", str(classes), str(scala)], check=True
+        )
+        outputs = sorted(classes.rglob("*"))
+        compiled = {path.relative_to(classes).as_posix(): path for path in outputs if path.is_file()}
+        if not compiled or any(not name.startswith("io/joern/jssrc2cpg/utils/AstGenRunner") for name in compiled):
+            raise ValueError("Compiler produced unexpected classes")
+        adapted = work / "adapted.jar"
+        with zipfile.ZipFile(jar) as original, zipfile.ZipFile(adapted, "w", zipfile.ZIP_DEFLATED) as destination:
+            for entry in original.infolist():
+                if entry.filename not in compiled:
+                    destination.writestr(entry, original.read(entry.filename))
+            for name, path in compiled.items():
+                destination.write(path, name)
+        provenance = root / "ousast-frontend-retention-v1"
+        provenance.mkdir(exist_ok=True)
+        for source_path in (file_utils, scala, script, astgen / "bun.lock"):
+            shutil.copy2(source_path, provenance / source_path.name)
+        binary = astgen / "astgen-linux-x64"
+        manifest = {
+            "revision": "ousast-frontend-retention-v1",
+            "builder_sha256": digest(Path(__file__)),
+            "policy": "OUSAST_INCLUDE_TESTS=1",
+            "sources": PINS,
+            "original_jar_sha256": JAR_SHA256,
+            "adapted_jar_sha256": digest(adapted),
+            "astgen_sha256": digest(binary),
+            "compiled_entries": sorted(compiled),
+        }
+        shutil.copy2(binary, root / "frontends/jssrc2cpg/bin/astgen/astgen-linux")
+        shutil.copy2(adapted, jar)
+        (provenance / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    build(Path(sys.argv[1]).resolve())

ops/joern.py

@@ -40,7 +40,7 @@ STAMP = f"{PREFIX}/.joern-version"
 # PHP-Parser), so without `php` on PATH a PHP tree fails to build with an opaque "Process exited with code 1"
 # -- measured on this machine before the package was added. `php_frontend` is opt-in because most targets do
 # not need it and it pulls a second runtime.
-PACKAGES = ["openjdk-21-jre-headless", "unzip"]
+PACKAGES = ["openjdk-21-jre-headless", "unzip", "python3", "curl", "ca-certificates"]
 if host.data.get("php_frontend", False):
     PACKAGES.append("php-cli")
 
@@ -101,3 +101,19 @@ server.shell(
     name="Verify joern-parse runs",
     commands=[f"{PREFIX}/joern-cli/joern-parse --help > /dev/null"],
 )
+
+# The same checksum-pinned source adaptation as the packaged image. Installing it is
+# explicit and fails on an unexpected engine; stock versions do not claim parity.
+files.put(
+    name="Upload the versioned frontend-retention builder",
+    src="ops/frontend-retention/install.py",
+    dest=f"{PREFIX}/frontend-retention-install.py",
+    _sudo=True,
+)
+server.shell(
+    name="Build the approved frontend-retention adaptation",
+    commands=[
+        f"python3 {PREFIX}/frontend-retention-install.py {PREFIX}/joern-cli",
+    ],
+    _sudo=True,
+)

ops/smoke_partitions.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import hashlib
 import json
+import os
 import sys
 import tempfile
 import time
@@ -17,6 +18,7 @@ from openultrasast.model.scan import ScanBudget, scan_repository
 
 
 def main() -> None:
+    include_tests = os.environ.get("OUSAST_SMOKE_INCLUDE_TESTS", "1") == "1"
     deadline = ExecutionBudget(time.monotonic() + 900, 2.0)
     records: list[dict[str, object]] = []
     with tempfile.TemporaryDirectory(prefix="ousast-partitions-smoke-") as temporary:
@@ -43,6 +45,8 @@ def main() -> None:
             "node_modules/library.js": "function vendor_js_marker(req) { eval(req); }\n",
             "unsupported.go": "package example\nfunc Unmodeled() {}\n",
         }
+        suffix_tests = {f"checks/probe.{suffix}.js": f"js_{suffix}_marker" for suffix in ("test", "spec", "mock", "e2e")}
+        sources.update({path: f"function {method}() {{ return 42; }}\n" for path, method in suffix_tests.items()})
         inputs = {}
         for filename, content in sources.items():
             path = root / filename
@@ -83,6 +87,9 @@ def main() -> None:
                     assert any(str(path).endswith("server/api.js") for path in files), witness
                     assert any(str(path).endswith("tests/probe.js") for path in files), witness
                     assert "js_handler" in methods and "js_test_marker" in methods, witness
+                    for path, method in suffix_tests.items():
+                        assert any(str(filename).endswith(path) for filename in files), witness
+                        assert method in methods, witness
                     assert "php_handler" not in methods, witness
                 else:
                     raise AssertionError(f"unexpected frontend: {language}")
@@ -100,7 +107,7 @@ def main() -> None:
         result = scan_repository(
             root,
             regions,
-            backend=RecordingBackend(heap_mb=1024),
+            backend=RecordingBackend(heap_mb=1024, include_tests=include_tests),
             budget=ScanBudget(max_regions=1),
             execution_budget=deadline,
             ranking_mode="static",
@@ -119,6 +126,7 @@ def main() -> None:
         report = {
             "purpose": "Real mixed PHP/JavaScript graph census and one global ranker budget; no admission or latency claim",
             "shared_budget_seconds": 900,
+            "include_tests": include_tests,
             "inputs": inputs,
             "graph_witnesses": records,
             "result": asdict(result),