mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
160 lines
7.7 KiB
Python
160 lines
7.7 KiB
Python
"""Build the approved test-retention adaptation from checksum-pinned upstream sources.
|
|
|
|
Usage: python3 install.py /opt/joern-cli. Requires curl, unzip, Java 21 and Python3.
|
|
Only the two upstream input filters change. OUSAST_INCLUDE_TESTS=1 opts in;
|
|
unset/0 keeps upstream behavior. Sources, lockfile and build identity are retained.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import urllib.request
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
PINS = {
|
|
"astgen.tar.gz": (
|
|
"https://codeload.github.com/joernio/astgen-monorepo/tar.gz/refs/tags/javascript-astgen/v3.50.1",
|
|
"8d9728dca8eab694a0f07bcd7a1c9a88368cb7bd354fc19c2ee8ca8611ff869a",
|
|
),
|
|
"AstGenRunner.scala": (
|
|
"https://raw.githubusercontent.com/joernio/joern/v4.0.625/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala",
|
|
"9963f949360d7e17baed1bd8875b717b2855745cf7260eef8a87d17533a54268",
|
|
),
|
|
"bun.zip": (
|
|
"https://github.com/oven-sh/bun/releases/download/bun-v1.4.2/bun-linux-x64.zip",
|
|
"36368faef7527875d5ffa52e53cd48021741f2a83eb6208a8dd64068d422a913",
|
|
),
|
|
}
|
|
JAR_SHA256 = "49fe488a79cc52b4e1d2e056143ec414f59e0007ed25f054c5fa7ae4aa34dff3"
|
|
|
|
|
|
def digest(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def replace_once(path: Path, before: str, after: str) -> None:
|
|
source = path.read_text()
|
|
if source.count(before) != 1:
|
|
raise ValueError(f"Unexpected source shape: {path.name}")
|
|
path.write_text(source.replace(before, after))
|
|
|
|
|
|
def build(root: Path) -> None:
|
|
jar = root / "frontends/jssrc2cpg/lib/io.joern.jssrc2cpg-4.0.625.jar"
|
|
installed = root / "ousast-frontend-retention-v1/manifest.json"
|
|
if installed.is_file():
|
|
previous = json.loads(installed.read_text())
|
|
if (
|
|
previous.get("builder_sha256") == digest(Path(__file__))
|
|
and previous.get("adapted_jar_sha256") == digest(jar)
|
|
and previous.get("astgen_sha256") == digest(root / "frontends/jssrc2cpg/bin/astgen/astgen-linux")
|
|
):
|
|
print("Verified installed frontend-retention adaptation")
|
|
return
|
|
if digest(jar) != JAR_SHA256:
|
|
raise ValueError("Expected pristine Joern 4.0.625 jssrc2cpg jar")
|
|
with tempfile.TemporaryDirectory(prefix="ousast-frontend-build-") as temporary:
|
|
work = Path(temporary)
|
|
for name, (url, expected) in PINS.items():
|
|
with urllib.request.urlopen(url) as response:
|
|
(work / name).write_bytes(response.read())
|
|
if digest(work / name) != expected:
|
|
raise ValueError(f"Checksum mismatch: {name}")
|
|
with tarfile.open(work / "astgen.tar.gz") as archive:
|
|
archive.extractall(work, filter="data")
|
|
with zipfile.ZipFile(work / "bun.zip") as archive:
|
|
archive.extractall(work)
|
|
bun = work / "bun-linux-x64/bun"
|
|
bun.chmod(0o755)
|
|
astgen = work / "astgen-monorepo-javascript-astgen-v3.50.1/javascript-astgen"
|
|
file_utils = astgen / "src/FileUtils.ts"
|
|
replace_once(
|
|
file_utils,
|
|
"const IGNORE_DIRS_SET = new Set(Defaults.IGNORE_DIRS.map((d) => d.toLowerCase()))",
|
|
"""const INCLUDE_TESTS = process.env.OUSAST_INCLUDE_TESTS === "1"
|
|
const TEST_DIRS = new Set(["test", "tests", "e2e", "e2e-beta", "cypress", "__tests__", "__mocks__"])
|
|
const IGNORE_DIRS_SET = new Set(Defaults.IGNORE_DIRS.filter(d => !INCLUDE_TESTS || !TEST_DIRS.has(d)).map(d => d.toLowerCase()))
|
|
const IGNORE_FILE_PATTERN = INCLUDE_TESTS
|
|
? new RegExp("(chunk-vendors|app~|conf|[.-]min|\\\\.d)\\\\.(js|jsx|cjs|mjs|xsjs|xsjslib|ts|tsx)$", "i")
|
|
: Defaults.IGNORE_FILE_PATTERN""",
|
|
)
|
|
replace_once(
|
|
file_utils,
|
|
'dirName.startsWith("__") ||',
|
|
'(dirName.startsWith("__") && !(INCLUDE_TESTS && TEST_DIRS.has(dirName.toLowerCase()))) ||',
|
|
)
|
|
replace_once(file_utils, "Defaults.IGNORE_FILE_PATTERN.test(fileName)", "IGNORE_FILE_PATTERN.test(fileName)")
|
|
scala = work / "AstGenRunner.scala"
|
|
replace_once(
|
|
scala,
|
|
"lazy val isIgnoredTest = IgnoredTestsRegex.exists(_.matches(filePath))",
|
|
'lazy val isIgnoredTest = !sys.env.get("OUSAST_INCLUDE_TESTS").contains("1") && IgnoredTestsRegex.exists(_.matches(filePath))',
|
|
)
|
|
# Keep the original Gruntfile path and bytes. Other default filters remain intact.
|
|
replace_once(
|
|
scala,
|
|
"lazy val isIgnored = IgnoredFilesRegex.exists(_.matches(filePath))",
|
|
"lazy val isIgnored = IgnoredFilesRegex.exists(_.matches(filePath)) && "
|
|
'!(sys.env.get("OUSAST_INCLUDE_BUILD_CONFIGS").contains("1") && '
|
|
'Paths.get(filePath).getFileName.toString == "Gruntfile.js")',
|
|
)
|
|
# Upstream stages embedded TS standard-library assets; retain that build recipe,
|
|
# selecting just the supported Linux x64 target instead of all release platforms.
|
|
script = astgen / "scripts/build-binaries.ts"
|
|
source = script.read_text()
|
|
start = source.index("const targets = [")
|
|
end = source.index("] as const", start) + len("] as const")
|
|
script.write_text(
|
|
source[:start] + 'const targets = [{target: "bun-linux-x64", outfile: "./astgen-linux-x64"}] as const' + source[end:]
|
|
)
|
|
subprocess.run([str(bun), "install", "--frozen-lockfile"], cwd=astgen, check=True)
|
|
subprocess.run([str(bun), "scripts/sync-version.ts"], cwd=astgen, check=True)
|
|
subprocess.run([str(bun), "scripts/build-binaries.ts"], cwd=astgen, check=True)
|
|
classes = work / "classes"
|
|
classes.mkdir()
|
|
jars = sorted(set(root.glob("lib/*.jar")) | set(root.glob("frontends/jssrc2cpg/lib/*.jar")))
|
|
classpath = ":".join(map(str, jars))
|
|
subprocess.run(
|
|
["java", "-cp", classpath, "dotty.tools.dotc.Main", "-classpath", classpath, "-d", str(classes), str(scala)], check=True
|
|
)
|
|
outputs = sorted(classes.rglob("*"))
|
|
compiled = {path.relative_to(classes).as_posix(): path for path in outputs if path.is_file()}
|
|
if not compiled or any(not name.startswith("io/joern/jssrc2cpg/utils/AstGenRunner") for name in compiled):
|
|
raise ValueError("Compiler produced unexpected classes")
|
|
adapted = work / "adapted.jar"
|
|
with zipfile.ZipFile(jar) as original, zipfile.ZipFile(adapted, "w", zipfile.ZIP_DEFLATED) as destination:
|
|
for entry in original.infolist():
|
|
if entry.filename not in compiled:
|
|
destination.writestr(entry, original.read(entry.filename))
|
|
for name, path in compiled.items():
|
|
destination.write(path, name)
|
|
provenance = root / "ousast-frontend-retention-v1"
|
|
provenance.mkdir(exist_ok=True)
|
|
for source_path in (file_utils, scala, script, astgen / "bun.lock"):
|
|
shutil.copy2(source_path, provenance / source_path.name)
|
|
binary = astgen / "astgen-linux-x64"
|
|
manifest = {
|
|
"revision": "ousast-frontend-retention-v1",
|
|
"builder_sha256": digest(Path(__file__)),
|
|
"policy": "OUSAST_INCLUDE_TESTS=1; OUSAST_INCLUDE_BUILD_CONFIGS=1 retains Gruntfile.js",
|
|
"sources": PINS,
|
|
"original_jar_sha256": JAR_SHA256,
|
|
"adapted_jar_sha256": digest(adapted),
|
|
"astgen_sha256": digest(binary),
|
|
"compiled_entries": sorted(compiled),
|
|
}
|
|
shutil.copy2(binary, root / "frontends/jssrc2cpg/bin/astgen/astgen-linux")
|
|
shutil.copy2(adapted, jar)
|
|
(provenance / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
build(Path(sys.argv[1]).resolve())
|