From 56cfc9da8916217cba0adf041453a68aaa301063 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:32:18 +0200 Subject: [PATCH] feat(pre-push-safety-net): reproduce evaluated Joern runtime --- .kiro/specs/pre-push-safety-net/tasks.md | 3 +- .kiro/steering/roadmap.md | 2 +- Dockerfile | 22 +++- .../2026-09-12-engine-runtime-smoke.json | 79 ++++++++++++ docker-compose.yml | 2 +- ops/README.md | 18 ++- ops/joern.py | 20 ++- ops/smoke_engine.py | 117 ++++++++++++++++++ 8 files changed, 252 insertions(+), 11 deletions(-) create mode 100644 benchmarks/measurements/2026-09-12-engine-runtime-smoke.json create mode 100644 ops/smoke_engine.py diff --git a/.kiro/specs/pre-push-safety-net/tasks.md b/.kiro/specs/pre-push-safety-net/tasks.md index a1af1c3..d533740 100644 --- a/.kiro/specs/pre-push-safety-net/tasks.md +++ b/.kiro/specs/pre-push-safety-net/tasks.md @@ -22,12 +22,13 @@ started with group 1 (tasks 1.1–1.4); each task remains pending until independ shared backend/driver, cache and report edits otherwise remain serial. No optional acceptance tests. - [ ] 1. Establish the runtime, evaluation inputs and shared contracts -- [ ] 1.1 Reproduce the evaluated engine environment +- [x] 1.1 Reproduce the evaluated engine environment - Align the shipped image, Compose and host-install defaults with Joern 4.0.625; preserve version override and checksum verification. - Exercise PHP with the native interpreter and the installed JavaScript frontend; open a source and the frontend parser where applicable, recording byte counts and graph census. - Done when a local image smoke produces valid small PHP and JavaScript graph/query payloads with recorded versions, and an unreadable source fails explicitly. This builds locally; it does not deploy. - _Boundary: Cache and Engine Boundary, packaging integration_ - _Requirements: 7.1, 7.3, 8.2_ + - _Verified 2026-09-12:_ independent review APPROVED; 888 tests passed, 9 skipped. Actual local image PHP/JavaScript graph census and source witnesses passed, unreadable PHP explicitly refused. Parent verified current provenance hashes, image identity, non-root user and CLI/frontend liveness. Evidence: `benchmarks/measurements/2026-09-12-engine-runtime-smoke.json`. Runtime readiness only; no hook latency/detection claim. Existing mypy errors in `model/scan.py` at 299/302/310 remain unchanged. - [ ] 1.2 Prepare labeled development and reserved evaluation changes - Pin a readable PMPro change/fix and a supported Node/Express change/fix, plus benign controls; verify license and source bytes before accepting recipes. - Use NodeGoat only if its selected mechanism fits a current arbiter; label authored repairs as authored. Reserve an independently selected Node API repository with modern module/async context and reviewable history before tuning on transfer results. diff --git a/.kiro/steering/roadmap.md b/.kiro/steering/roadmap.md index 1b4ed39..8ff3eaa 100644 --- a/.kiro/steering/roadmap.md +++ b/.kiro/steering/roadmap.md @@ -25,7 +25,7 @@ engine are out of scope. VAmPI 16. `<50` remains unmet; VAmPI's transitive SQLI is not measured by this position metric. - Equal-score tie-breaking alone cannot bring PMPro or WP Statistics under 50. The existing ranker is useful, but its measured ordering is not enabled by the ordinary CLI caller. -- Docker/Compose/ops still default to 4.0.623. Pin and provenance alignment precede deployment claims. +- Docker/Compose/ops now default to 4.0.625; task 1.1 passed local PHP/JavaScript image smoke and independent review. No host deployment or hook latency claim follows from that runtime evidence. - `contributor-scan` has 24/40 checked tasks; historical checkboxes are not rewritten by this review. `flow-aware-ranking` is still an unapproved brief despite implementation history. diff --git a/Dockerfile b/Dockerfile index 030a149..b30f8d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ FROM eclipse-temurin:21-jre-noble -ARG JOERN_VERSION=v4.0.623 +ARG JOERN_VERSION=v4.0.625 ARG ARCHIVE=joern-cli-linux-x86_64.zip # php-cli is required by php2cpg, which drives PHP-Parser and shells out to a real interpreter. Without it a @@ -25,12 +25,19 @@ RUN set -eux; \ cd /tmp; \ curl -fsSL -O "https://github.com/joernio/joern/releases/download/${JOERN_VERSION}/${ARCHIVE}"; \ curl -fsSL -O "https://github.com/joernio/joern/releases/download/${JOERN_VERSION}/${ARCHIVE}.sha512"; \ - sha512sum -c "${ARCHIVE}.sha512"; \ + sed 's@ target/@ @' "${ARCHIVE}.sha512" > checksum.sha512; \ + sha512sum -c checksum.sha512; \ unzip -q "${ARCHIVE}" -d /opt; \ - rm -f "${ARCHIVE}" "${ARCHIVE}.sha512"; \ + rm -f "${ARCHIVE}" "${ARCHIVE}.sha512" checksum.sha512; \ /opt/joern-cli/joern-parse --help > /dev/null ENV PATH="/opt/joern-cli:${PATH}" +# This release ships the JS launcher with a .sh suffix; the backend's direct fallback uses +# the same extensionless command convention as the other frontends. +RUN if [ ! -e /opt/joern-cli/jssrc2cpg ] && [ -x /opt/joern-cli/jssrc2cpg.sh ]; then \ + ln -s jssrc2cpg.sh /opt/joern-cli/jssrc2cpg; \ + fi + WORKDIR /app COPY pyproject.toml README.md ./ COPY src ./src @@ -43,7 +50,14 @@ RUN python3 -m venv /venv \ ENV PATH="/venv/bin:${PATH}" # Analysis never needs root. -RUN useradd --create-home --uid 1000 ousast && chown -R ousast /app +# Noble may already provide the ubuntu account at UID 1000. Reuse that identity rather than +# failing installation or creating a second account with the same UID. +RUN if getent passwd 1000 > /dev/null; then \ + usermod --login ousast --home /home/ousast --move-home "$(getent passwd 1000 | cut -d: -f1)"; \ + else \ + useradd --create-home --uid 1000 ousast; \ + fi \ + && chown -R ousast /app USER ousast ENTRYPOINT ["python", "-m", "openultrasast.cli"] diff --git a/benchmarks/measurements/2026-09-12-engine-runtime-smoke.json b/benchmarks/measurements/2026-09-12-engine-runtime-smoke.json new file mode 100644 index 0000000..0d428d9 --- /dev/null +++ b/benchmarks/measurements/2026-09-12-engine-runtime-smoke.json @@ -0,0 +1,79 @@ +{ + "joern_version": "4.0.625", + "php_version": "PHP 8.3.6 (cli) (built: Sep 2 2026 12:56:02) (NTS)", + "php_parser_bytes": 43, + "purpose": "runtime and query smoke; does not qualify vulnerability detection or hook latency", + "languages": { + "php": { + "frontend": "php2cpg", + "source_bytes": 45, + "source_sha256": "f744d636071c3e5a8e72ef33fa746db22e705dfdc66dba98b5238875054d801a", + "graph_bytes": 49508, + "census": { + "files": "2", + "methods": "3", + "overlays": "base,controlflow,typerel,callgraph,dataflowOss", + "maxHeapMB": "1024" + }, + "source_witness": { + "files": [ + "probe.php", + "" + ], + "handlers": 1, + "evalCalls": 1 + }, + "elapsed_seconds": 52.295 + }, + "javascript": { + "frontend": "jssrc2cpg", + "source_bytes": 48, + "source_sha256": "10c0c21a6e2f988119f305809af1f98e1ff39fecb9f40d6cd3dfa05301e0015b", + "graph_bytes": 71163, + "census": { + "files": "1", + "methods": "5", + "overlays": "base,controlflow,typerel,callgraph,dataflowOss", + "maxHeapMB": "1024" + }, + "source_witness": { + "files": [ + "probe.js" + ], + "handlers": 1, + "evalCalls": 1 + }, + "elapsed_seconds": 32.334 + } + }, + "unreadable_source": { + "status": "explicit_failure", + "reason": "the php interpreter cannot read /php/probe.php, so the frontend would parse nothing and write an empty graph. Check that php is a real interpreter with access to this tree -- a container or wrapper without the right mount fails exactly this way." + }, + "recorded_at": "2026-09-12T16:28:54.206163+00:00", + "task": "pre-push-safety-net 1.1", + "command": "docker run --rm --network none --memory 3g --entrypoint python -v \"$PWD/ops/smoke_engine.py:/smoke_engine.py:ro\" openultrasast:dev /smoke_engine.py", + "image_id": "sha256:5566d669c91bdce380275f22ae9d4aa7587866cd7b5568d9f772e20f35e41696", + "base_commit": "7e3b3bbc8b95b0bf947b5e9746382fcf9d8cfdb8", + "working_tree_file_sha256": { + "Dockerfile": "f0c7e3f4ec15553694063622b88c1b3da4875ad80e248c932f6dd88e48741a2f", + "docker-compose.yml": "1fc07e61b3156e0f31c6d2aa48e24c36349cf37aba373f4c83a0329cbc4d7b47", + "ops/joern.py": "30ac85e5e7b8431b9d1bc11a8074f77489077f6c5a636e11b6fb320fe5cd4d00", + "ops/smoke_engine.py": "f4d41b010f56f2b59e44a253e3c8e49dc6275eff903c1441de181e568011648a", + "src/openultrasast/cpg/backend.py": "2971f83d491b3e739e32ae3a1869cb92c138bde93ef7ea4ba5329c5a34301fdc", + "src/openultrasast/cpg/queries/census.sc": "d18964575fe8dd065871bdc453aa12a61857ad78b2a2d4c13e0fc6e2648736e3", + "src/openultrasast/cpg/queries/overlay.sc": "0b76e56b76b52c33ca61d7541ea0f8ebeb61088f1ab0866e9e42f93d7b67f644" + }, + "path_note": "Ephemeral input scratch paths replaced with ; these are smoke file witnesses, not stable graph/cache identities.", + "validation": { + "build": "docker compose build ousast: exit 0; upstream archive SHA512 verified before extraction", + "backend_regressions": "40 passed in 2.98s (tests/test_cpg_backend.py)", + "packaging_checks": "Ruff check/format passed for changed Python; git diff --check passed; Compose default and override resolved", + "checksum_controls": "target-prefix and bare filename controls accepted; corrupted archive bytes rejected" + }, + "resolved_packaging_failures": [ + "Upstream checksum names target/, now normalized to downloaded basename without changing hash.", + "Base image already owns UID1000; reuse its account as ousast, preserving non-root UID1000.", + "Installed JS launcher is jssrc2cpg.sh; add extensionless alias for backend fallback." + ] +} diff --git a/docker-compose.yml b/docker-compose.yml index ff45b3f..0d19357 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: build: context: . args: - JOERN_VERSION: ${JOERN_VERSION:-v4.0.623} + JOERN_VERSION: ${JOERN_VERSION:-v4.0.625} image: openultrasast:${OUSAST_VERSION:-dev} volumes: - ${TARGET:-.}:/target:ro # the repository under analysis, read-only diff --git a/ops/README.md b/ops/README.md index 6994d64..7b17d68 100644 --- a/ops/README.md +++ b/ops/README.md @@ -61,13 +61,29 @@ Pin the version and its checksum in an inventory rather than editing the deploy: ```python # inventory.py -hosts = [("@local", {"joern_version": "v4.0.623", "joern_prefix": "/opt/joern"})] +hosts = [("@local", {"joern_version": "v4.0.625", "joern_prefix": "/opt/joern"})] ``` The checksum needs no inventory entry: the deploy fetches the `.sha512` upstream publishes beside the archive and runs `sha512sum -c` against it, before unpacking. (pyinfra's `files.download` offers sha384/sha256/sha1/md5 but not sha512, which is what Joern publishes — so verification is an explicit shell step rather than a download argument.) +The upstream checksum's `target/` path prefix is removed to match the local download path; +the digest remains unchanged and a mismatch still stops installation. + +Verify an image locally after building it (no model endpoint or network is needed for the smoke): + +```bash +docker run --rm --network none --memory 3g --entrypoint python \ + -v "$PWD/ops/smoke_engine.py:/smoke_engine.py:ro" \ + openultrasast:dev /smoke_engine.py > engine-smoke.json +``` + +This checks native PHP source/parser readability, installed PHP and JavaScript frontends, saved +dataflow overlays, graph census and exact source-node witnesses, and explicit refusal of an unreadable +PHP file. It prints JSON only after all checks pass; a failed build/query raises an error with a nonzero +exit. Versions, source hashes/byte counts and timings describe this small runtime smoke, not detection +quality or a pre-push latency guarantee. Keep the image's default non-root user for the permission test. Why this exists: Joern was first installed here by hand with `curl` and `unzip`, which is fine once and unreproducible thereafter. The engine decides whether a finding is a `suspicion` or a `model_entailed`, and diff --git a/ops/joern.py b/ops/joern.py index ac7d4f6..d0387b4 100644 --- a/ops/joern.py +++ b/ops/joern.py @@ -30,7 +30,7 @@ from pyinfra.operations import files, server # Pinned. An engine that silently changes version changes every verdict it produces, and the ceiling # artifacts under benchmarks/measurements are only comparable against a fixed one. -JOERN_VERSION = host.data.get("joern_version", "v4.0.623") +JOERN_VERSION = host.data.get("joern_version", "v4.0.625") PREFIX = host.data.get("joern_prefix", "/opt/joern") ARCHIVE = "joern-cli-linux-x86_64.zip" BASE = f"https://github.com/joernio/joern/releases/download/{JOERN_VERSION}" @@ -71,12 +71,26 @@ server.shell( curl -fsSL -o {ARCHIVE} {BASE}/{ARCHIVE} curl -fsSL -o {ARCHIVE}.sha512 {BASE}/{ARCHIVE}.sha512 # Verify before unpacking, never after: the archive is what we are choosing to trust. - sha512sum -c {ARCHIVE}.sha512 + # Upstream names target/; our download is directly in /tmp. + sed 's@ target/@ @' {ARCHIVE}.sha512 > checksum.sha512 + sha512sum -c checksum.sha512 rm -rf {PREFIX}/joern-cli unzip -q -o {ARCHIVE} -d {PREFIX} echo {JOERN_VERSION} > {STAMP} # 1.8 GB in /tmp is how a disk fills up. - rm -f {ARCHIVE} {ARCHIVE}.sha512 + rm -f {ARCHIVE} {ARCHIVE}.sha512 checksum.sha512 + fi + """ + ], + _sudo=True, +) + +server.shell( + name="Expose the JavaScript frontend under the backend's command name", + commands=[ + f""" + if [ ! -e {PREFIX}/joern-cli/jssrc2cpg ] && [ -x {PREFIX}/joern-cli/jssrc2cpg.sh ]; then + ln -s jssrc2cpg.sh {PREFIX}/joern-cli/jssrc2cpg fi """ ], diff --git a/ops/smoke_engine.py b/ops/smoke_engine.py new file mode 100644 index 0000000..d7d4492 --- /dev/null +++ b/ops/smoke_engine.py @@ -0,0 +1,117 @@ +"""Exercise the installed engine, not a mock. Run inside the shipped image as its non-root user.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +from openultrasast.cpg.backend import DATAFLOW_OVERLAY, JoernBackend, joern_version + + +def require(condition: object, message: str) -> None: + if not condition: + raise RuntimeError(message) + + +def main() -> None: + require(os.getuid() != 0, "run as the image's non-root user so unreadable-file checks are meaningful") + version = joern_version() + require(version, "installed Joern version is unknown") + php = Path(shutil.which("php") or "/missing-php").resolve() + require(php.read_bytes()[:4] == b"\x7fELF", "PHP must be a native executable, not a stdio wrapper") + parser = Path(shutil.which("php2cpg") or "/missing-php2cpg").resolve().parent + parser /= "frontends/php2cpg/bin/php-parser/php-parser.php" + probe = "$s = file_get_contents($argv[1]); if ($s === false || strlen($s) === 0) exit(3); echo strlen($s);" + + def php_bytes(path: Path) -> int: + result = subprocess.run([str(php), "-r", probe, str(path)], check=True, capture_output=True, text=True, timeout=15) + return int(result.stdout) + + report: dict[str, object] = { + "joern_version": ".".join(map(str, version)), + "php_version": subprocess.run([str(php), "-v"], check=True, capture_output=True, text=True, timeout=15).stdout.splitlines()[0], + "php_parser_bytes": php_bytes(parser), + "purpose": "runtime and query smoke; does not qualify vulnerability detection or hook latency", + "languages": {}, + } + languages = report["languages"] + assert isinstance(languages, dict) + with tempfile.TemporaryDirectory(prefix="ousast-engine-smoke-") as temporary: + root = Path(temporary) + queries = root / "queries" + queries.mkdir() + (queries / "source_witness.sc").write_text("""@main def exec(cpgFile: String) = { + importCpg(cpgFile) + println("---OUSAST-CPG-BEGIN---") + println(ujson.write(ujson.Obj( + "files" -> ujson.Arr.from(cpg.file.name.l), + "handlers" -> cpg.method.nameExact("handler").size, + "evalCalls" -> cpg.call.nameExact("eval").size + ))) + println("---OUSAST-CPG-END---") +} +""") + for language, frontend, filename, content in ( + ("php", "php2cpg", "probe.php", " 0 and int(census["methods"]) > 0, f"empty {language} census") + require(DATAFLOW_OVERLAY in str(census["overlays"]).split(","), f"{language} missing dataflow overlay: {census}") + witness = JoernBackend(queries_dir=queries, query_timeout=180, heap_mb=1024).query(graph.cpg_path, "source_witness", {}) + require(isinstance(witness, dict), f"{language} source witness unanswered") + assert isinstance(witness, dict) + require(any(str(path).endswith(filename) for path in witness["files"]), f"{language} source absent from graph") + require(witness["handlers"] > 0 and witness["evalCalls"] > 0, f"{language} expected source nodes missing: {witness}") + languages[language] = { + "frontend": frontend, + "source_bytes": len(data), + "source_sha256": hashlib.sha256(data).hexdigest(), + "graph_bytes": graph.cpg_path.stat().st_size, + "census": census, + "source_witness": witness, + "elapsed_seconds": round(time.monotonic() - started, 3), + } + print(f"{language}: census and source witness verified", file=sys.stderr, flush=True) + finally: + if graph.cleanup: + graph.cleanup() + if language == "php": + source.chmod(0) + try: + refused = backend.build(source_root, language=language) + require(refused is None and "cannot read" in backend.last_failure, "unreadable source did not fail explicitly") + report["unreadable_source"] = {"status": "explicit_failure", "reason": backend.last_failure} + finally: + source.chmod(0o600) + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main()