From 46232fe3e6b13fb50a1ae3dc1c778e0fcecb0a7e Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:36:10 +0200 Subject: [PATCH] feat(ops): ship the tool and its engine in one image A contributor should be able to run this with nothing installed but Docker: docker compose build TARGET=/path/to/repo docker compose run --rm ousast scan /target One image rather than two services, and the reason is the seam. The model layer reaches Joern by subprocess; putting both in the same image means Joern is simply on PATH, so there is no container-to-container call, no docker socket to mount, no host/container path translation, and no second code path in cpg/backend.py to get wrong. Sibling services would need the tool to exec into the engine's container, which means handing a linter the docker socket -- a large amount of privilege for something that reads source code. The image is big (a JRE plus ~2 GB of Joern); that is the honest cost of bundling an engine. I had started building the two-service version with a docker-wrapper mode in the backend and dropped it, tests included: once the engine is on PATH the wrapper is complexity with no purpose. Defaults chosen so an unfamiliar analyser is safe to try: the target mounts READ-ONLY, /work is a tmpfs so CPG scratch never outlives the run or touches the host, network_mode is `none` unless the LLM judge is wanted, and API keys pass through the environment rather than into an image layer. mem_limit is set because two Joern JVMs at once is what triggered an out-of-memory kill on a 7.7 GB machine during this feature's measurements. pyinfra deploy kept for persistent host installs, with the same version pin and the same checksum verification -- done as an explicit `sha512sum -c` step because pyinfra's files.download offers sha384/sha256/sha1/md5 and Joern publishes sha512. That mismatch was caught by validating against the installed pyinfra rather than trusting the API from memory. Both include php-cli: php2cpg drives PHP-Parser and shells out to a real interpreter, and without it a PHP tree fails with an opaque "Process exited with code 1" -- measured here, which is what makes a WordPress corpus feasible. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- Dockerfile | 50 ++++++++++++++++++++++++++ docker-compose.yml | 33 +++++++++++++++++ ops/README.md | 82 ++++++++++++++++++++++++++++++++++++++++++ ops/joern.py | 89 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 254 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 ops/README.md create mode 100644 ops/joern.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..030a149 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,50 @@ +# OpenUltraSAST with its CPG engine, in one image. +# +# The model layer's arbiter is Joern, and `openultrasast.cpg.backend` reaches it by subprocess. Shipping both +# in one image is what makes that simple: Joern is on PATH inside, so there is no container-to-container call, +# no docker socket, no path translation between host and container, and no second code path in the backend to +# get wrong. The alternative -- tool and engine as sibling services -- needs the tool to exec into the other +# container, which means mounting the docker socket, and that is a large amount of privilege for a linter. +# +# The image is big (a JRE plus ~2 GB of Joern). That is the honest cost of bundling an engine, and it buys a +# contributor a working tool from `docker compose run` with nothing installed on their machine. + +FROM eclipse-temurin:21-jre-noble + +ARG JOERN_VERSION=v4.0.623 +ARG ARCHIVE=joern-cli-linux-x86_64.zip + +# php-cli is required by php2cpg, which drives PHP-Parser and shells out to a real interpreter. Without it a +# PHP tree fails with an opaque "Process exited with code 1" -- measured on this project, not assumed. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-pip unzip curl ca-certificates git php-cli \ + && rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + cd /tmp; \ + curl -fsSL -O "https://github.com/joernio/joern/releases/download/${JOERN_VERSION}/${ARCHIVE}"; \ + curl -fsSL -O "https://github.com/joernio/joern/releases/download/${JOERN_VERSION}/${ARCHIVE}.sha512"; \ + sha512sum -c "${ARCHIVE}.sha512"; \ + unzip -q "${ARCHIVE}" -d /opt; \ + rm -f "${ARCHIVE}" "${ARCHIVE}.sha512"; \ + /opt/joern-cli/joern-parse --help > /dev/null +ENV PATH="/opt/joern-cli:${PATH}" + +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY src ./src +COPY benchmarks ./benchmarks + +# The core install is zero-dependency by design; `semantic` adds the tree-sitter grammars the candidate +# enumerator uses. The LLM endpoint stays optional and is configured by environment, never baked in. +RUN python3 -m venv /venv \ + && /venv/bin/pip install --no-cache-dir -e ".[semantic]" +ENV PATH="/venv/bin:${PATH}" + +# Analysis never needs root. +RUN useradd --create-home --uid 1000 ousast && chown -R ousast /app +USER ousast + +ENTRYPOINT ["python", "-m", "openultrasast.cli"] +CMD ["--help"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d9e0eb8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +# OpenUltraSAST as a contributor runs it: nothing installed on the host but Docker. +# +# docker compose build +# TARGET=/path/to/repo docker compose run --rm ousast scan /target +# +# The target is mounted READ-ONLY. A tool that reads your source to tell you about it has no business being +# able to write to it, and a contributor evaluating an unfamiliar analyser should not have to take that on +# trust. + +services: + ousast: + build: + context: . + args: + JOERN_VERSION: ${JOERN_VERSION:-v4.0.623} + image: openultrasast:${OUSAST_VERSION:-dev} + volumes: + - ${TARGET:-.}:/target:ro # the repository under analysis, read-only + - ./reports:/reports # where findings land, the one writable mount + working_dir: /work + tmpfs: + # CPG construction is scratch-heavy and none of it should outlive the run or touch the host. + - /work:size=${WORK_SIZE:-4g} + environment: + # The LLM endpoint is optional: without it the model layer still entails, and the suspicion band simply + # goes unasked. Passed through rather than baked in, so no key is ever in an image layer. + DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY:-} + OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} + TMPDIR: /work + # A CPG build is memory-hungry, and two Joern JVMs at once is what triggered an out-of-memory kill on a + # 7.7 GB machine during this project's measurements. + mem_limit: ${OUSAST_MEM_LIMIT:-6g} + network_mode: ${OUSAST_NETWORK:-none} # `bridge` only when the LLM judge is wanted diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 0000000..21f3fce --- /dev/null +++ b/ops/README.md @@ -0,0 +1,82 @@ +# ops + +Infrastructure for the optional engines. Nothing here is a project dependency: the core install is +`dependencies = []` and every engine below degrades to a recorded reason when it is absent. + +## Two ways to get the engine + +**Docker (recommended for contributors).** The tool and Joern ship in one image, so nothing is installed on +the host and there is no container-to-container call, no docker socket, and no host/container path +translation: + +```bash +docker compose build +TARGET=/path/to/repo docker compose run --rm ousast scan /target +``` + +The target is mounted read-only; `/work` is a tmpfs so CPG scratch never outlives the run. The LLM endpoint +is optional and passed through the environment, never baked into a layer — without it the model layer still +entails and only the `suspicion` band goes unasked. `network_mode` defaults to `none`; set it to `bridge` +only when you want the judge. + +**pyinfra (for a persistent host install).** Same engine, same pin, same checksum, but on the machine: + +```bash +uv pip install pyinfra # tooling only +pyinfra @local ops/joern.py # this machine +``` + +Pin the version and its checksum in an inventory rather than editing the deploy: + +```python +# inventory.py +hosts = [("@local", {"joern_version": "v4.0.623", "joern_prefix": "/opt/joern"})] +``` + +The checksum needs no inventory entry: the deploy fetches the `.sha512` upstream publishes beside the +archive and runs `sha512sum -c` against it, before unpacking. (pyinfra's `files.download` offers +sha384/sha256/sha1/md5 but not sha512, which is what Joern publishes — so verification is an explicit +shell step rather than a download argument.) + +Why this exists: Joern was first installed here by hand with `curl` and `unzip`, which is fine once and +unreproducible thereafter. The engine decides whether a finding is a `suspicion` or a `model_entailed`, and +every ceiling number in `benchmarks/measurements` is only comparable against a fixed version — so the version +is pinned, the download is checksum-verified, and a re-run with a matching stamp is a no-op. + +It installs an engine; it does not become one. No sandbox, no container, no service. `openultrasast.cpg.backend` +reaches Joern by subprocess and this changes nothing about that. + +## What we actually use from Joern + +A small, deliberately narrow surface — worth stating, because it is what determines whether a given Joern +version works for us: + +| entry point | use | +|---|---| +| `joern-parse --output cpg.bin` | build the CPG | +| `joern --script q.sc --param k=v` | run a shipped CPGQL query, JSON fenced on stdout | + +CPGQL steps, all of them in `src/openultrasast/cpg/queries/*.sc`: + +| step | why | +|---|---| +| `cpg.call` / `cpg.method` / `cpg.identifier` | locate sinks, operations, sources | +| `.reachableByFlows` | the interprocedural dataflow the flat IR could not do | +| `.argument`, `.argumentIndexGt(0)` | arity and the safe-shape test — index 0 is the receiver, named args are −1 | +| `.ast.isLiteral` | constant abstraction; a keyword argument is assignment-shaped so the literal sits in its subtree | +| `.dominatedBy`, `.controlledBy` | guard dominance for absence bugs; `controlledBy` is what catches a check that runs *after* the fetch | +| `.controlStructure.condition` | identity branches | +| `.method.filename` / `.lineNumber` / `.lineNumberEnd` | closure scoping by line-range containment | +| `.methodFullName` | resolved sink matching | +| `ujson` | the fenced JSON payload | + +No Joern server mode, no workspace/project management, no `joern-scan`, no overlays beyond the defaults. + +## Languages + +Joern ships frontends for C/C++, Java (source and bytecode), JavaScript/TypeScript, Python, Kotlin, Go, C#, +PHP, ABAP and Ghidra binaries. We currently model **python, javascript, java, c**. + +**PHP works but needs a PHP interpreter**: `php2cpg` drives PHP-Parser and shells out to `php`, so on a host +without it a PHP tree fails to build with an opaque "Process exited with code 1". Set `php_frontend: True` in +the inventory to install `php-cli` alongside the JVM. diff --git a/ops/joern.py b/ops/joern.py new file mode 100644 index 0000000..ac7d4f6 --- /dev/null +++ b/ops/joern.py @@ -0,0 +1,89 @@ +"""pyinfra deploy for the Joern CPG engine (model-grounded-detection, Req 4). + +Replaces the ad-hoc `curl | unzip` this engine was first installed with. Joern is the model layer's arbiter +substrate and the only thing standing between a `suspicion` and a `model_entailed`, so how it reaches a +machine should be reproducible and stated rather than remembered. + +Deliberately narrow: + +* **It installs an engine, it does not become one.** No sandbox, no container, no service. + ``openultrasast.cpg.backend`` reaches Joern by subprocess and nothing here changes that. +* **The checksum is verified, in a shell step rather than by ``files.download``.** Upstream publishes a + ``.sha512`` beside the archive and pyinfra's download operation offers sha384/sha256/sha1/md5 but not + sha512, so the verification is done with ``sha512sum -c`` against the published file. An unverified 1.8 GB + download that then parses your source tree is not something to wave through. +* **Idempotent.** A version stamp short-circuits the download and extraction, so a re-run costs a file read. +* **Nothing the tool needs at runtime.** The core install stays ``dependencies = []``; without Joern the model + layer degrades to ``suspicion`` with a recorded reason, which is a supported state rather than a broken one. + +Usage: + + uv pip install pyinfra # tooling only, never a project dependency + pyinfra @local ops/joern.py # this machine + pyinfra inventory.py ops/joern.py # anywhere else +""" + +from __future__ import annotations + +from pyinfra import host +from pyinfra.operations import files, server + +# Pinned. An engine that silently changes version changes every verdict it produces, and the ceiling +# artifacts under benchmarks/measurements are only comparable against a fixed one. +JOERN_VERSION = host.data.get("joern_version", "v4.0.623") +PREFIX = host.data.get("joern_prefix", "/opt/joern") +ARCHIVE = "joern-cli-linux-x86_64.zip" +BASE = f"https://github.com/joernio/joern/releases/download/{JOERN_VERSION}" +STAMP = f"{PREFIX}/.joern-version" + +# Joern ships Scala but not a JVM. `php2cpg` additionally shells out to a real PHP interpreter (it drives +# PHP-Parser), so without `php` on PATH a PHP tree fails to build with an opaque "Process exited with code 1" +# -- measured on this machine before the package was added. `php_frontend` is opt-in because most targets do +# not need it and it pulls a second runtime. +PACKAGES = ["openjdk-21-jre-headless", "unzip"] +if host.data.get("php_frontend", False): + PACKAGES.append("php-cli") + +server.packages( + name="Install a JVM, unzip, and (optionally) the PHP frontend's interpreter", + packages=PACKAGES, + present=True, + _sudo=True, +) + +files.directory( + name=f"Create {PREFIX}", + path=PREFIX, + present=True, + _sudo=True, +) + +server.shell( + name=f"Install Joern {JOERN_VERSION} (skipped when the stamp already matches)", + commands=[ + # One guarded block: a matching stamp skips the download, the checksum and the extraction alike. + f""" + if [ -f {STAMP} ] && grep -qx {JOERN_VERSION} {STAMP}; then + echo "joern {JOERN_VERSION} already installed" + else + set -e + cd /tmp + curl -fsSL -o {ARCHIVE} {BASE}/{ARCHIVE} + curl -fsSL -o {ARCHIVE}.sha512 {BASE}/{ARCHIVE}.sha512 + # Verify before unpacking, never after: the archive is what we are choosing to trust. + sha512sum -c {ARCHIVE}.sha512 + rm -rf {PREFIX}/joern-cli + unzip -q -o {ARCHIVE} -d {PREFIX} + echo {JOERN_VERSION} > {STAMP} + # 1.8 GB in /tmp is how a disk fills up. + rm -f {ARCHIVE} {ARCHIVE}.sha512 + fi + """ + ], + _sudo=True, +) + +server.shell( + name="Verify joern-parse runs", + commands=[f"{PREFIX}/joern-cli/joern-parse --help > /dev/null"], +)