feat(contributor-scan): retain first-party JavaScript tests in graphs

2026-10-16 03:49:26 +00:00 · 2026-09-13 11:29:48 +02:00
parent d02406e758
commit 51fabef9ea
15 changed files with 1329 additions and 26 deletions
@@ -0,0 +1,36 @@
+# First-party JavaScript test retention
+
+This source adaptation implements contributor-scan task 2.18 for the pinned Linux
+x86-64 Joern 4.0.625 installation. ASTGen 3.50.1 excludes test directories and
+filenames; jssrc2cpg additionally excludes test suffixes. Both filters must accept
+the explicit inclusion policy for a retained source file to reach the graph.
+
+`install.py` verifies pinned upstream source and tool archives and the original
+jssrc2cpg jar, rebuilds ASTGen with its frozen dependency lockfile, and compiles
+the adapted Scala source against Joern's shipped compiler and libraries. The
+changes affect test filtering only. Original input paths, bytes and imports are
+preserved. Vendor and user exclusions still apply.
+
+Docker and `ops/joern.py` run the same installer. A manual installation requires
+Python 3.12 or later, Java 21, network access for the pinned build inputs, and a
+pristine matching Joern installation:
+
+```bash
+python3 ops/frontend-retention/install.py /opt/joern-cli
+```
+
+The installer writes `ousast-frontend-retention-v1/manifest.json` under the Joern
+root, including builder, dependency and output hashes. A repeated invocation
+verifies the installed outputs and needs no network. If the recipe or installed
+outputs change, reinstall the pristine pinned engine before rebuilding; the
+installer refuses to layer an unverified adaptation onto modified jars.
+
+The adapted frontend keeps stock behavior unless `OUSAST_INCLUDE_TESTS=1`.
+`JoernBackend` explicitly enables it by default; `include_tests=False` supplies
+the regression control. Direct frontend users must set the variable themselves.
+Engine provenance must include the installed adaptation and effective policy:
+the upstream version string alone cannot distinguish these graphs.
+
+After building the image, run `ops/smoke_partitions.py` as described in `ops/README.md`.
+Its off control uses `OUSAST_SMOKE_INCLUDE_TESTS=0` and must fail the test-retention
+assertion. First-party test retention does not imply security capability admission.
@@ -0,0 +1,151 @@
+"""Build the approved test-retention adaptation from checksum-pinned upstream sources.
+
+Usage: python3 install.py /opt/joern-cli. Requires curl, unzip, Java 21 and Python3.
+Only the two upstream input filters change. OUSAST_INCLUDE_TESTS=1 opts in;
+unset/0 keeps upstream behavior. Sources, lockfile and build identity are retained.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import urllib.request
+import zipfile
+from pathlib import Path
+
+PINS = {
+    "astgen.tar.gz": (
+        "https://codeload.github.com/joernio/astgen-monorepo/tar.gz/refs/tags/javascript-astgen/v3.50.1",
+        "8d9728dca8eab694a0f07bcd7a1c9a88368cb7bd354fc19c2ee8ca8611ff869a",
+    ),
+    "AstGenRunner.scala": (
+        "https://raw.githubusercontent.com/joernio/joern/v4.0.625/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala",
+        "9963f949360d7e17baed1bd8875b717b2855745cf7260eef8a87d17533a54268",
+    ),
+    "bun.zip": (
+        "https://github.com/oven-sh/bun/releases/download/bun-v1.4.2/bun-linux-x64.zip",
+        "36368faef7527875d5ffa52e53cd48021741f2a83eb6208a8dd64068d422a913",
+    ),
+}
+JAR_SHA256 = "49fe488a79cc52b4e1d2e056143ec414f59e0007ed25f054c5fa7ae4aa34dff3"
+
+
+def digest(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def replace_once(path: Path, before: str, after: str) -> None:
+    source = path.read_text()
+    if source.count(before) != 1:
+        raise ValueError(f"Unexpected source shape: {path.name}")
+    path.write_text(source.replace(before, after))
+
+
+def build(root: Path) -> None:
+    jar = root / "frontends/jssrc2cpg/lib/io.joern.jssrc2cpg-4.0.625.jar"
+    installed = root / "ousast-frontend-retention-v1/manifest.json"
+    if installed.is_file():
+        previous = json.loads(installed.read_text())
+        if (
+            previous.get("builder_sha256") == digest(Path(__file__))
+            and previous.get("adapted_jar_sha256") == digest(jar)
+            and previous.get("astgen_sha256") == digest(root / "frontends/jssrc2cpg/bin/astgen/astgen-linux")
+        ):
+            print("Verified installed frontend-retention adaptation")
+            return
+    if digest(jar) != JAR_SHA256:
+        raise ValueError("Expected pristine Joern 4.0.625 jssrc2cpg jar")
+    with tempfile.TemporaryDirectory(prefix="ousast-frontend-build-") as temporary:
+        work = Path(temporary)
+        for name, (url, expected) in PINS.items():
+            with urllib.request.urlopen(url) as response:
+                (work / name).write_bytes(response.read())
+            if digest(work / name) != expected:
+                raise ValueError(f"Checksum mismatch: {name}")
+        with tarfile.open(work / "astgen.tar.gz") as archive:
+            archive.extractall(work, filter="data")
+        with zipfile.ZipFile(work / "bun.zip") as archive:
+            archive.extractall(work)
+        bun = work / "bun-linux-x64/bun"
+        bun.chmod(0o755)
+        astgen = work / "astgen-monorepo-javascript-astgen-v3.50.1/javascript-astgen"
+        file_utils = astgen / "src/FileUtils.ts"
+        replace_once(
+            file_utils,
+            "const IGNORE_DIRS_SET = new Set(Defaults.IGNORE_DIRS.map((d) => d.toLowerCase()))",
+            """const INCLUDE_TESTS = process.env.OUSAST_INCLUDE_TESTS === "1"
+const TEST_DIRS = new Set(["test", "tests", "e2e", "e2e-beta", "cypress", "__tests__", "__mocks__"])
+const IGNORE_DIRS_SET = new Set(Defaults.IGNORE_DIRS.filter(d => !INCLUDE_TESTS || !TEST_DIRS.has(d)).map(d => d.toLowerCase()))
+const IGNORE_FILE_PATTERN = INCLUDE_TESTS
+    ? new RegExp("(chunk-vendors|app~|conf|[.-]min|\\\\.d)\\\\.(js|jsx|cjs|mjs|xsjs|xsjslib|ts|tsx)$", "i")
+    : Defaults.IGNORE_FILE_PATTERN""",
+        )
+        replace_once(
+            file_utils,
+            'dirName.startsWith("__") ||',
+            '(dirName.startsWith("__") && !(INCLUDE_TESTS && TEST_DIRS.has(dirName.toLowerCase()))) ||',
+        )
+        replace_once(file_utils, "Defaults.IGNORE_FILE_PATTERN.test(fileName)", "IGNORE_FILE_PATTERN.test(fileName)")
+        scala = work / "AstGenRunner.scala"
+        replace_once(
+            scala,
+            "lazy val isIgnoredTest = IgnoredTestsRegex.exists(_.matches(filePath))",
+            'lazy val isIgnoredTest = !sys.env.get("OUSAST_INCLUDE_TESTS").contains("1") && IgnoredTestsRegex.exists(_.matches(filePath))',
+        )
+        # Upstream stages embedded TS standard-library assets; retain that build recipe,
+        # selecting just the supported Linux x64 target instead of all release platforms.
+        script = astgen / "scripts/build-binaries.ts"
+        source = script.read_text()
+        start = source.index("const targets = [")
+        end = source.index("] as const", start) + len("] as const")
+        script.write_text(
+            source[:start] + 'const targets = [{target: "bun-linux-x64", outfile: "./astgen-linux-x64"}] as const' + source[end:]
+        )
+        subprocess.run([str(bun), "install", "--frozen-lockfile"], cwd=astgen, check=True)
+        subprocess.run([str(bun), "scripts/sync-version.ts"], cwd=astgen, check=True)
+        subprocess.run([str(bun), "scripts/build-binaries.ts"], cwd=astgen, check=True)
+        classes = work / "classes"
+        classes.mkdir()
+        jars = sorted(set(root.glob("lib/*.jar")) | set(root.glob("frontends/jssrc2cpg/lib/*.jar")))
+        classpath = ":".join(map(str, jars))
+        subprocess.run(
+            ["java", "-cp", classpath, "dotty.tools.dotc.Main", "-classpath", classpath, "-d", str(classes), str(scala)], check=True
+        )
+        outputs = sorted(classes.rglob("*"))
+        compiled = {path.relative_to(classes).as_posix(): path for path in outputs if path.is_file()}
+        if not compiled or any(not name.startswith("io/joern/jssrc2cpg/utils/AstGenRunner") for name in compiled):
+            raise ValueError("Compiler produced unexpected classes")
+        adapted = work / "adapted.jar"
+        with zipfile.ZipFile(jar) as original, zipfile.ZipFile(adapted, "w", zipfile.ZIP_DEFLATED) as destination:
+            for entry in original.infolist():
+                if entry.filename not in compiled:
+                    destination.writestr(entry, original.read(entry.filename))
+            for name, path in compiled.items():
+                destination.write(path, name)
+        provenance = root / "ousast-frontend-retention-v1"
+        provenance.mkdir(exist_ok=True)
+        for source_path in (file_utils, scala, script, astgen / "bun.lock"):
+            shutil.copy2(source_path, provenance / source_path.name)
+        binary = astgen / "astgen-linux-x64"
+        manifest = {
+            "revision": "ousast-frontend-retention-v1",
+            "builder_sha256": digest(Path(__file__)),
+            "policy": "OUSAST_INCLUDE_TESTS=1",
+            "sources": PINS,
+            "original_jar_sha256": JAR_SHA256,
+            "adapted_jar_sha256": digest(adapted),
+            "astgen_sha256": digest(binary),
+            "compiled_entries": sorted(compiled),
+        }
+        shutil.copy2(binary, root / "frontends/jssrc2cpg/bin/astgen/astgen-linux")
+        shutil.copy2(adapted, jar)
+        (provenance / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    build(Path(sys.argv[1]).resolve())