diff --git a/.github/workflows/engine-image.yml b/.github/workflows/engine-image.yml index 078dc75..a1ea234 100644 --- a/.github/workflows/engine-image.yml +++ b/.github/workflows/engine-image.yml @@ -3,18 +3,11 @@ on: workflow_dispatch: push: branches: [main] + tags: ['v*'] paths: - - 'src/**' - - 'benchmarks/learn/engine_trace*.py' - - 'benchmarks/learn/engine_task_entry.py' - - 'benchmarks/unseen/replay_entry.py' - - 'benchmarks/learn/engine_worker_service.py' + - 'plane/Dockerfile.*' + - 'plane/task-base.digest' - 'Dockerfile' - - 'plane/Dockerfile.engine-task' - - 'plane/Dockerfile.search-task' - - 'ops/frontend-retention/install.py' - - 'pyproject.toml' - - '.github/workflows/engine-image.yml' permissions: contents: read packages: write @@ -22,7 +15,40 @@ concurrency: group: engine-image-${{ github.ref }} cancel-in-progress: false jobs: + pins: + runs-on: ubuntu-latest + outputs: + base_tag: ${{ steps.pins.outputs.base_tag }} + base_digest: ${{ steps.pins.outputs.base_digest }} + browser_tag: ${{ steps.pins.outputs.browser_tag }} + browser_digest: ${{ steps.pins.outputs.browser_digest }} + steps: + - uses: actions/checkout@v5 + - uses: docker/setup-buildx-action@v3 + # No login: every pinned base must be anonymously pullable. + - id: pins + run: | + python3 - <<'PYTHON' + import json, os, re, subprocess + from pathlib import Path + pins = json.loads(Path("plane/task-base.digest").read_text()) + manifests = {} + for key, output in (("base", "base"), ("base-browser", "browser")): + ref = pins.get(key) + pattern = r"ghcr.io/norandom/ousast-task-base:(base-[a-z0-9-]+)@(sha256:[0-9a-f]{64})" + match = re.fullmatch(pattern, ref or "") + if not match: + raise SystemExit(f"{key}: unpublished/invalid pin; run task-base-image and update plane/task-base.digest") + raw = subprocess.check_output(["docker", "buildx", "imagetools", "inspect", "--raw", ref]) + manifests[key] = json.loads(raw)["layers"] + with open(os.environ["GITHUB_OUTPUT"], "a") as out: + out.write(f"{output}_tag={match[1]}\n{output}_digest={match[2]}\n") + base = manifests["base"] + if not base or manifests["base-browser"][:len(base)] != base: + raise SystemExit("Browser pin does not share the pinned base's layers") + PYTHON image: + needs: pins runs-on: ubuntu-latest strategy: fail-fast: false @@ -30,12 +56,6 @@ jobs: include: - package: openultrasast dockerfile: Dockerfile - - package: ousast-engine-task - dockerfile: plane/Dockerfile.engine-task - target: engine - - package: ousast-replay-task - dockerfile: plane/Dockerfile.engine-task - target: replay - package: ousast-search-task dockerfile: plane/Dockerfile.search-task steps: @@ -58,9 +78,18 @@ jobs: with: context: . file: ${{ matrix.dockerfile }} - target: ${{ matrix.target }} platforms: linux/amd64 push: true + provenance: false + build-args: | + TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }} + TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }} + TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }} + TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }} + cache-from: | + type=gha,scope=${{ matrix.package }} + type=gha,scope=ousast-engine-task + cache-to: type=gha,mode=max,scope=${{ matrix.package }} tags: | ${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }} ${{ steps.names.outputs.image }}:main @@ -70,3 +99,101 @@ jobs: run: | echo "$IMAGE@$DIGEST" echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY" + docker logout ghcr.io + docker buildx imagetools inspect "$IMAGE@$DIGEST" + engine: + needs: pins + runs-on: ubuntu-latest + outputs: + image: ${{ steps.names.outputs.image }} + digest: ${{ steps.build.outputs.digest }} + steps: + - uses: actions/checkout@v5 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: names + env: + OWNER: ${{ github.repository_owner }} + PACKAGE: ousast-engine-task + run: | + echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT" + echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" + - id: build + uses: docker/build-push-action@v6 + with: + context: . + file: plane/Dockerfile.engine-task + target: engine + platforms: linux/amd64 + push: true + provenance: false + build-args: | + TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }} + TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }} + TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }} + TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }} + cache-from: type=gha,scope=ousast-engine-task + cache-to: type=gha,mode=max,scope=ousast-engine-task + tags: | + ${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }} + ${{ steps.names.outputs.image }}:main + - env: + IMAGE: ${{ steps.names.outputs.image }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + echo "$IMAGE@$DIGEST" + echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY" + docker logout ghcr.io + docker buildx imagetools inspect "$IMAGE@$DIGEST" + replay: + needs: [pins, engine] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: names + env: + OWNER: ${{ github.repository_owner }} + PACKAGE: ousast-replay-task + run: | + echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT" + echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" + - id: build + uses: docker/build-push-action@v6 + with: + context: . + file: plane/Dockerfile.engine-task + target: replay + platforms: linux/amd64 + push: true + provenance: false + build-args: | + ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }} + TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }} + TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }} + TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }} + TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }} + cache-from: | + type=gha,scope=ousast-replay-task + type=gha,scope=ousast-engine-task + cache-to: type=gha,mode=max,scope=ousast-replay-task + tags: | + ${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }} + ${{ steps.names.outputs.image }}:main + - env: + IMAGE: ${{ steps.names.outputs.image }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + echo "$IMAGE@$DIGEST" + echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY" + docker logout ghcr.io + docker buildx imagetools inspect "$IMAGE@$DIGEST" diff --git a/.github/workflows/task-base-image.yml b/.github/workflows/task-base-image.yml new file mode 100644 index 0000000..e41977f --- /dev/null +++ b/.github/workflows/task-base-image.yml @@ -0,0 +1,78 @@ +name: Task base image +on: + workflow_dispatch: + push: + branches: [main] + paths: + - 'plane/Dockerfile.base' + - 'pyproject.toml' + - 'uv.lock' +permissions: + contents: read + packages: write +concurrency: + group: task-base-image + cancel-in-progress: false +jobs: + base: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: names + run: | + echo "tag=base-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" + content=$(cat plane/Dockerfile.base pyproject.toml uv.lock | sha256sum | cut -d' ' -f1) + echo "content=base-content-$content" >> "$GITHUB_OUTPUT" + - id: base + uses: docker/build-push-action@v6 + with: + context: . + file: plane/Dockerfile.base + target: base + platforms: linux/amd64 + push: true + provenance: false + sbom: false + tags: | + ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }} + ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }} + cache-from: type=gha,scope=task-base + cache-to: type=gha,mode=max,scope=task-base + - id: browser + uses: docker/build-push-action@v6 + with: + context: . + file: plane/Dockerfile.base + target: base-browser + platforms: linux/amd64 + push: true + provenance: false + sbom: false + build-args: | + TASK_BASE_REF=ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }} + tags: | + ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser + ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }}-browser + cache-from: type=gha,scope=task-base-browser + cache-to: type=gha,mode=max,scope=task-base-browser + - name: Print manual pin update and verify public access + env: + BASE: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }} + BROWSER: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser@${{ steps.browser.outputs.digest }} + run: | + python3 - <<'PYTHON' >> "$GITHUB_STEP_SUMMARY" + import json, os + print("Replace plane/task-base.digest with this JSON after both public checks pass:") + print("```json") + print(json.dumps({"base": os.environ["BASE"], "base-browser": os.environ["BROWSER"]}, indent=2)) + print("```") + PYTHON + docker logout ghcr.io + docker buildx imagetools inspect "$BASE" + docker buildx imagetools inspect "$BROWSER" diff --git a/Dockerfile b/Dockerfile index 6062c23..59deac8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,14 @@ -# OpenUltraSAST with its CPG engine, in one image. -# -# The model layer's arbiter is Joern, and `openultrasast.cpg.backend` reaches it by subprocess. Shipping both -# in one image is what makes that simple: Joern is on PATH inside, so there is no container-to-container call, -# no docker socket, no path translation between host and container, and no second code path in the backend to -# get wrong. The alternative -- tool and engine as sibling services -- needs the tool to exec into the other -# container, which means mounting the docker socket, and that is a large amount of privilege for a linter. -# -# The image is big (a JRE plus ~2 GB of Joern). That is the honest cost of bundling an engine, and it buys a -# contributor a working tool from `docker compose run` with nothing installed on their machine. - -FROM eclipse-temurin:21-jre-noble -LABEL org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST" - +# syntax=docker/dockerfile:1 +# Supply digests from plane/task-base.digest; deliberately no floating fallback. +ARG TASK_BASE_TAG=base-unpublished +ARG TASK_BASE_DIGEST +FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS joern-build +ENV PATH="/opt/joern-cli:${PATH}" ARG JOERN_VERSION=v4.0.625 ARG ARCHIVE=joern-cli-linux-x86_64.zip -# php-cli is required by php2cpg, which drives PHP-Parser and shells out to a real interpreter. Without it a -# PHP tree fails with an opaque "Process exited with code 1" -- measured on this project, not assumed. RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 python3-venv python3-pip unzip curl ca-certificates git php-cli \ + && apt-get install -y --no-install-recommends curl unzip \ && rm -rf /var/lib/apt/lists/* RUN set -eux; \ @@ -32,39 +20,28 @@ RUN set -eux; \ unzip -q "${ARCHIVE}" -d /opt; \ rm -f "${ARCHIVE}" "${ARCHIVE}.sha512" checksum.sha512; \ /opt/joern-cli/joern-parse --help > /dev/null -ENV PATH="/opt/joern-cli:${PATH}" -# This release ships the JS launcher with a .sh suffix; the backend's direct fallback uses -# the same extensionless command convention as the other frontends. RUN if [ ! -e /opt/joern-cli/jssrc2cpg ] && [ -x /opt/joern-cli/jssrc2cpg.sh ]; then \ ln -s jssrc2cpg.sh /opt/joern-cli/jssrc2cpg; \ fi - COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \ && rm /tmp/frontend-retention-install.py -WORKDIR /app -COPY pyproject.toml README.md ./ -COPY src ./src -COPY benchmarks ./benchmarks - -# The core install is zero-dependency by design; `semantic` adds the tree-sitter grammars the candidate -# enumerator uses. The LLM endpoint stays optional and is configured by environment, never baked in. -RUN python3 -m venv /venv \ - && /venv/bin/pip install --no-cache-dir -e ".[semantic,s3]" -ENV PATH="/venv/bin:${PATH}" - -# Analysis never needs root. -# Noble may already provide the ubuntu account at UID 1000. Reuse that identity rather than -# failing installation or creating a second account with the same UID. +FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS engine +ENV PATH="/opt/joern-cli:${PATH}" +COPY --from=joern-build /opt/joern-cli /opt/joern-cli +# Create runtime identity before the application layers. RUN if getent passwd 1000 > /dev/null; then \ usermod --login ousast --home /home/ousast --move-home "$(getent passwd 1000 | cut -d: -f1)"; \ - else \ - useradd --create-home --uid 1000 ousast; \ - fi \ - && chown -R ousast /app + else useradd --create-home --uid 1000 ousast; fi +WORKDIR /app +COPY pyproject.toml README.md LICENSE ./ +COPY src ./src +RUN --mount=type=cache,target=/root/.cache/pip \ + /venv/bin/pip install --no-deps --no-cache-dir . +COPY --chown=1000 benchmarks ./benchmarks +RUN chown -R ousast /app USER ousast - ENTRYPOINT ["python", "-m", "openultrasast.cli"] CMD ["--help"] diff --git a/ops/ax/README.md b/ops/ax/README.md index e1be9c3..526903b 100644 --- a/ops/ax/README.md +++ b/ops/ax/README.md @@ -260,3 +260,102 @@ filesystem quota; programs that hardcode `/tmp` still consume the worker's RAM. Repository clones must use HTTPS `git clone https://github.com/...git` or codeload archives, never `api.github.com`: tasks share one public IP. Search task inputs are presigned checkout archives, so these entrypoints do not clone repositories themselves. + +## Image layers, publication and node cache budget + +The task family has one shared runtime base, `ousast-task-base`, built from the +public AX runner v0.3.1. `base-browser` adds Chromium to that exact published base +(by digest); engine and replay do not carry Chromium. The VM/kind `openultrasast` +image uses the same base and Joern recipe. Replay inherits the published engine +digest in CI, so even a cold build cannot accidentally duplicate its Joern layers. + +Java is **Temurin 21 JDK**, copied from `eclipse-temurin:21-jdk-jammy`. +`ops/frontend-retention/install.py` explicitly requires Java 21 and invokes +`java -cp ... dotty.tools.dotc.Main` using Joern's bundled Scala compiler. +The shared JDK also provides `javac` for Maven/Gradle project builds. +Curl/unzip and retention build scratch stay in a discarded Joern build stage. +Joern stays above the base in the engine image (also used by replay and the VM +lane); its compiler jars and retention provenance must not be blindly pruned. +Runtime dependencies from `pyproject.toml` (core plus semantic/s3 for the VM lane) +precede project code. Package installation uses `--no-deps --no-cache-dir`; +its isolated packaging backend is temporary. + +The base retains upstream Python 3.12 with venv/pip. Its explicit apt package list +is `nodejs npm node-typescript php-cli composer maven sqlite3 bubblewrap util-linux +git ca-certificates libstdc++6 zlib1g`. Composer is the trixie apt package. +Chromium is installed only in `base-browser`. Apt caches/lists and docs/man/info +are cleaned; dpkg excludes new docs/man/info. Files already in inherited AX +layers still occupy space even if deleted. + +Executor preparation supports pip, npm, Composer, Maven and Gradle builds. +There is **no installed Gradle binary**: the executor invokes the selected project's +`gradlew` through `/bin/sh`, including for wrappers without executable permissions. +Gradle-wrapper distribution downloads and dependency downloads go through +**public HTTPS from the executor**, subject to its egress policy and deadline. +The verifier receives prepared products and performs no dependency downloads. + +Rebuild policy: + +- Dependency/toolchain changes: run **Task base image** (also triggered by changes + to `plane/Dockerfile.base`, `pyproject.toml`, or `uv.lock`). It publishes + `base-` and `base-content-` tags, plus their `-browser` + variants, and prints immutable digest references. Content tags identify build + inputs; they do not claim reproducible upstream apt/PyPI resolution. The lockfile + triggers refreshes but pip installs the declared runtime constraints, not a uv + frozen environment. Deliberate upstream refreshes use manual dispatch. +- After both targets publish, ensure `ousast-task-base` is **public** in GHCR's + package settings. Copy the workflow summary's JSON into `plane/task-base.digest` + and submit that pin update for normal review. CI does not commit on your behalf. + The initial null pins mean **unpublished**, not a usable bootstrap image. +- A pin update or `plane/Dockerfile.*`/root Dockerfile change on main runs **Engine + image**. For source, entrypoint or retention-installer changes, dispatch it once + for the desired revision, or push a `v*` release tag. Source-only merges no longer + publish images automatically. Never rebuild per task/run. All builds use scoped + GHA caches; pin checks precede builds and require anonymous access plus identical + base-layer ancestry for the browser variant. +- Final package names must also be public in GHCR. Publication checks anonymous + access after logout and fails if the package is private. Set visibility on first + publication and rerun as needed. Deploy only the reported `image@sha256:...` + references in the cluster profile/Task manifests, never `main` or a release tag. + Publication does not automatically change deployed Task pins. + +For a local task build, pass `TASK_BASE_TAG` and `TASK_BASE_DIGEST` from the base +pin; search instead takes `TASK_BROWSER_TAG` and `TASK_BROWSER_DIGEST`. Replay's +`ENGINE_REF` defaults to its local engine stage; set it to the published engine +reference to reuse exactly those layers. When building `base-browser` locally, +pass `TASK_BASE_REF=` for the same ancestry as CI. + +The operator's capacity rule is **all images in rotation, unpacked, together +at most approximately 9 GiB**, including rollback revisions. Working builds take +priority over minimizing an individual executor image; its previous 2.13 GB +unpacked (~1.98 GiB) was acceptable. Retire unused Task templates and image +revisions before rotating a large base/Joern generation. Shared layers occupy +storage once where the runtime deduplicates them, but do not assume that turns +a sum above 9 GiB into an acceptable rotation. Record both the sum of unpacked +image sizes and actual unique-layer storage. Never prune images used by live tasks. + +Expected footprint (planning estimates only; no images built for this change): + +| Image | Expected unpacked size | Basis / uncertainty | +| --- | --- | --- | +| Shared base | about 1.3–1.9 GiB | ~454 MB inherited AX layers + ~281 MB JDK baseline + npm/Composer/Maven, language runtimes and venv/extras; apt dependency closure unmeasured | +| Browser base | about 1.65–2.45 GiB | Shared base + roughly 350–550 MB Chromium/dependencies | +| Search executor/verifier | about 1.7–2.5 GiB | Browser base + application; previous 2.13 GB unpacked is a baseline, not a ceiling | +| Engine | roughly 3.5–6 GiB | Restored shared base + Joern; the supplied 2.16 GB compressed engine cannot establish its unpacked size | +| Replay | engine + a few KiB | Same engine digest plus replay entrypoint | +| VM/kind openultrasast | roughly 3.5–6 GiB plus benchmark assets | Same base/Joern, runtime extras shared, benchmark payload retained | + +An engine + search rotation therefore sums to about **5.2–8.5 GiB** before +rollback images. Adding replay as another unpacked image brings the sum to about +**8.7–14.5 GiB**; this rotation is not demonstrated to fit 9 GiB. Engine/replay +share almost all layers, but the operator must confirm the accounting and measured +footprint before selecting a rotation. Extra revisions and the VM image add to +the budget if retained there. + +The previous search history included a 1.37 GB apt layer and 281 MB JDK; no +per-package apt breakdown is available. Removing system Gradle and docs/caches +may save space, while retained build tools and shared Python extras cost space. +These ranges are not measured savings or a capacity guarantee. After publication, +measure unpacked images and unique layers for the entire intended rotation, and +verify build recipes, PHP/JavaScript retention, runner startup and Chromium before +rollout. There is no separate <1.5 GiB executor gate. diff --git a/plane/Dockerfile.base b/plane/Dockerfile.base new file mode 100644 index 0000000..7062b5a --- /dev/null +++ b/plane/Dockerfile.base @@ -0,0 +1,41 @@ +# syntax=docker/dockerfile:1 +# base-browser is built against the published base digest in CI; local builds +# default to the base stage. Never independently rebuild its shared ancestry. +ARG TASK_BASE_REF=base +FROM eclipse-temurin:21-jdk-jammy AS jdk + +FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS base +LABEL org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST" +USER root +COPY --from=jdk /opt/java/openjdk /opt/java/openjdk +ENV JAVA_HOME=/opt/java/openjdk +ENV PATH="/venv/bin:/opt/java/openjdk/bin:${PATH}" +# Use upstream /usr/local Python 3.12 (also required by frontend retention). +# Its venv/pip and the toolchains below support every executor build recipe. +# Gradle projects use their wrapper; no system Gradle is installed. +RUN printf 'path-exclude=/usr/share/doc/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/info/*\n' \ + > /etc/dpkg/dpkg.cfg.d/01-no-docs \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + nodejs npm node-typescript php-cli composer maven sqlite3 bubblewrap util-linux git ca-certificates libstdc++6 zlib1g \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /usr/share/man/* /usr/share/info/* \ + /opt/java/openjdk/lib/src.zip /opt/java/openjdk/man /root/.cache +# Metadata only: install runtime dependencies, including the VM lane's extras. +# No project package or packaging backend enters this shared environment. +COPY pyproject.toml /tmp/dependencies/pyproject.toml +RUN python3 -c 'import pathlib,tomllib; p=tomllib.loads(pathlib.Path("/tmp/dependencies/pyproject.toml").read_text())["project"]; print("\n".join(p["dependencies"] + p["optional-dependencies"]["semantic"] + p["optional-dependencies"]["s3"]))' > /tmp/requirements.txt \ + && python3 -m venv /venv \ + && /venv/bin/pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm -rf /tmp/dependencies /tmp/requirements.txt /root/.cache \ + && python3 --version && java -version && javac -version && node --version && npm --version && tsc --version \ + && php --version && composer --version && mvn --version && sqlite3 --version && bwrap --version \ + && setpriv --version && prlimit --version && nsenter --version + +FROM ${TASK_BASE_REF} AS base-browser +RUN apt-get update \ + && apt-get install -y --no-install-recommends chromium \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /usr/share/man/* /usr/share/info/* /root/.cache \ + && chromium --version diff --git a/plane/Dockerfile.engine-task b/plane/Dockerfile.engine-task index 23a5298..cf2702b 100644 --- a/plane/Dockerfile.engine-task +++ b/plane/Dockerfile.engine-task @@ -1,23 +1,15 @@ -# Build from the repository root. AX v0.3.1 starts spec.command immediately; -# preserve its runner as PID 1 and put the engine command in the Task manifest. -FROM eclipse-temurin:21-jre-jammy AS jre - -FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS engine -LABEL org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST" - -USER root +# syntax=docker/dockerfile:1 +# Supply digests from plane/task-base.digest; deliberately no floating fallback. +ARG TASK_BASE_TAG=base-unpublished +ARG TASK_BASE_DIGEST +ARG ENGINE_REF=engine +FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS joern-build +ENV PATH="/opt/joern-cli:${PATH}" ARG JOERN_VERSION=v4.0.625 ARG ARCHIVE=joern-cli-linux-x86_64.zip -# The AX runner is Python 3.12 / Debian slim. Copy the Java 21 runtime from -# Jammy (older glibc than Debian bookworm) instead of relying on distro JRE versions. -COPY --from=jre /opt/java/openjdk /opt/java/openjdk -ENV JAVA_HOME=/opt/java/openjdk -ENV PATH="/venv/bin:/opt/java/openjdk/bin:/opt/joern-cli:${PATH}" RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - php-cli unzip curl ca-certificates libstdc++6 zlib1g \ + && apt-get install -y --no-install-recommends curl unzip \ && rm -rf /var/lib/apt/lists/* RUN set -eux; \ @@ -37,24 +29,20 @@ COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \ && rm /tmp/frontend-retention-install.py +FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS engine +ENV PATH="/opt/joern-cli:${PATH}" +COPY --from=joern-build /opt/joern-cli /opt/joern-cli WORKDIR /app COPY pyproject.toml README.md LICENSE ./ COPY src ./src +RUN --mount=type=cache,target=/root/.cache/pip \ + /venv/bin/pip install --no-deps --no-cache-dir . COPY benchmarks/learn/engine_task_entry.py \ benchmarks/learn/engine_trace_worker.py \ benchmarks/learn/engine_trace_parse.py ./benchmarks/learn/ -# Worker imports are standard-library plus engine modules. Isolate them from -# the AX runner's Python environment and omit PyYAML, boto3 and all extras. -RUN python3 -m venv /venv \ - && /venv/bin/pip install --no-cache-dir --no-deps . \ - && /venv/bin/python -I -c 'import pathlib, runpy, sys; sys.modules.update(yaml=None, boto3=None); sys.path.insert(0, "/app/benchmarks/learn"); p = pathlib.Path("/app/benchmarks/learn/engine_trace_worker.py"); print("Worker import smoke bytes:", len(p.read_bytes())); runpy.run_path(str(p), run_name="engine_worker_import_smoke")' +RUN /venv/bin/python -I -c 'import pathlib, runpy, sys; sys.modules.update(yaml=None, boto3=None); sys.path.insert(0, "/app/benchmarks/learn"); p = pathlib.Path("/app/benchmarks/learn/engine_trace_worker.py"); print("Worker import smoke bytes:", len(p.read_bytes())); runpy.run_path(str(p), run_name="engine_worker_import_smoke")' +# ENTRYPOINT and CMD belong to the upstream AX runner. -# Do not set ENTRYPOINT or CMD: both belong to the upstream AX runner. - -FROM engine AS replay -RUN apt-get update \ - && apt-get install -y --no-install-recommends git \ - && rm -rf /var/lib/apt/lists/* +FROM ${ENGINE_REF} AS replay COPY benchmarks/unseen/replay_entry.py ./benchmarks/unseen/replay_entry.py -RUN /venv/bin/pip install --no-cache-dir . \ - && /venv/bin/ousast pre-push --help > /dev/null +RUN /venv/bin/ousast pre-push --help > /dev/null diff --git a/plane/Dockerfile.search-task b/plane/Dockerfile.search-task index 947b271..bed3b0d 100644 --- a/plane/Dockerfile.search-task +++ b/plane/Dockerfile.search-task @@ -1,35 +1,19 @@ -# Build from the repository root; retain the AX runner entrypoint and command. -FROM eclipse-temurin:17-jdk-jammy AS jdk -FROM ghcr.io/norandom/ax-task-runner:v0.3.1 -LABEL org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST" +# syntax=docker/dockerfile:1 +# Supply digests from plane/task-base.digest; deliberately no floating fallback. +ARG TASK_BROWSER_TAG=base-unpublished-browser +ARG TASK_BROWSER_DIGEST +FROM ghcr.io/norandom/ousast-task-base:${TASK_BROWSER_TAG}@${TASK_BROWSER_DIGEST} USER root -COPY --from=jdk /opt/java/openjdk /opt/java/openjdk -ENV JAVA_HOME=/opt/java/openjdk -ENV PATH="/venv/bin:/opt/java/openjdk/bin:${PATH}" -# dpkg exclusions prevent docs/manpages entering layers, including transitive deps. -RUN printf 'path-exclude=/usr/share/doc/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/info/*\n' \ - > /etc/dpkg/dpkg.cfg.d/01-no-docs \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 python3-venv python3-pip nodejs npm node-typescript php-cli composer maven gradle \ - sqlite3 bubblewrap util-linux chromium git ca-certificates \ - && rm -rf /var/lib/apt/lists/* /opt/java/openjdk/lib/src.zip /opt/java/openjdk/man -WORKDIR /app -COPY pyproject.toml README.md LICENSE ./ -COPY src ./src -# Includes the verifier-owned recipes, oracles, and key-free executor entrypoints. -RUN python3 -m venv /venv \ - && /venv/bin/pip install --no-cache-dir . \ - && python3 --version && node --version && tsc --version && php --version \ - && javac -version && sqlite3 --version && bwrap --version \ - && setpriv --version && prlimit --version && nsenter --version && chromium --version # Dedicated identity for artefacts, apps and Chromium; the verifier keeps root. RUN groupadd --gid 10001 search \ && useradd --create-home --uid 10001 --gid 10001 search -# gVisor is the outer boundary; retain inner bwrap when its capability probe succeeds. -# The Python base image sets GPG_KEY to the public CPython release-signing fingerprint (not a secret). -# Clear it so the executor's guard against inherited credentials stays strict and still passes. +# Upstream's public CPython signing fingerprint trips the credential guard. ENV GPG_KEY= +WORKDIR /app +COPY pyproject.toml README.md LICENSE ./ +COPY src ./src +RUN --mount=type=cache,target=/root/.cache/pip \ + /venv/bin/pip install --no-deps --no-cache-dir . +# Entrypoints and verifier-owned recipes/oracles are included in src. USER root -# Do not replace the AX runner's ENTRYPOINT or CMD. +# Preserve the AX runner ENTRYPOINT and CMD. diff --git a/plane/task-base.digest b/plane/task-base.digest new file mode 100644 index 0000000..96f9b97 --- /dev/null +++ b/plane/task-base.digest @@ -0,0 +1,5 @@ +{ + "base": null, + "base-browser": null, + "note": "UNPUBLISHED: run task-base-image, then replace with the two full tag@sha256 references from its summary. Never invent digests." +} diff --git a/src/openultrasast/search/executor.py b/src/openultrasast/search/executor.py index 0bf5e10..6817756 100644 --- a/src/openultrasast/search/executor.py +++ b/src/openultrasast/search/executor.py @@ -274,9 +274,28 @@ class InProcessExecutor: } def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]: + process_limit = None + try: + uid = os.getuid() + current = 0 + for entry in Path("/proc").iterdir(): + if not entry.name.isdecimal(): + continue + try: + status = (entry / "status").read_text() + except FileNotFoundError: + continue + real_uid = int(next(line for line in status.splitlines() if line.startswith("Uid:")).split()[1]) + current += real_uid == uid + process_limit = current + 128 + except (OSError, ValueError, IndexError, StopIteration): + pass + def constrain() -> None: resource.setrlimit(resource.RLIMIT_FSIZE, (limits.get("disk_bytes", task_storage.DEFAULT_SCRATCH_BYTES),) * 2) - resource.setrlimit(resource.RLIMIT_NPROC, (128, 128)) + # NPROC counts all processes of the real uid, so an absolute 128 blocks builds on busy hosts. + if process_limit is not None: + resource.setrlimit(resource.RLIMIT_NPROC, (process_limit,) * 2) resource.setrlimit(resource.RLIMIT_AS, (limits.get("memory_bytes", 256 * 1024**2),) * 2) resource.setrlimit(resource.RLIMIT_CPU, (math.ceil(timeout) + 1,) * 2) @@ -391,7 +410,10 @@ class InProcessExecutor: if recipe["recipe"] == "maven": argv += ["package", "-DskipTests"] elif recipe["recipe"] == "gradle": - argv += ["assemble"] + # The project pins its Gradle version through its wrapper. + # Invoke with sh because checkout archives may lose execute bits. + wrapper = self._path(str(path.relative_to(self.repo) / "gradlew")) + argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"] built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES}) if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]: return {**built, "status": "could_not_build"} diff --git a/tests/test_search_image.py b/tests/test_search_image.py index b1807f6..51000ac 100644 --- a/tests/test_search_image.py +++ b/tests/test_search_image.py @@ -4,17 +4,17 @@ import yaml def test_image_matches_runner_and_publishes_digest(): - engine = Path("plane/Dockerfile.engine-task").read_text() + base = Path("plane/Dockerfile.base").read_text() image = Path("plane/Dockerfile.search-task").read_text() - runner = "FROM ghcr.io/norandom/ax-task-runner:v0.3.1" - assert runner in engine and runner in image - for package in ("python3", "nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"): - assert package in image + assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1" in base + for package in ("nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"): + assert package in base + assert "python3 -m venv /venv" in base assert image.rsplit("USER ", 1)[1].startswith("root\n") assert "--uid 10001 --gid 10001 search" in image - assert "setpriv --version" in image - assert "17-jdk" in image and "--no-install-recommends" in image - assert "rm -rf /var/lib/apt/lists/" in image + assert "setpriv --version" in base + assert "21-jdk" in base and "--no-install-recommends" in base + assert "rm -rf /var/lib/apt/lists/" in base workflow_text = Path(".github/workflows/engine-image.yml").read_text() entries = yaml.safe_load(workflow_text)["jobs"]["image"]["strategy"]["matrix"]["include"] assert {"package": "ousast-search-task", "dockerfile": "plane/Dockerfile.search-task"} in entries diff --git a/tests/test_search_task_storage.py b/tests/test_search_task_storage.py index 3a45e4f..c78d6ba 100644 --- a/tests/test_search_task_storage.py +++ b/tests/test_search_task_storage.py @@ -2,7 +2,9 @@ import json import os +import signal import sys +import time from pathlib import Path import pytest @@ -222,3 +224,60 @@ def test_scratch_failure_preserves_exit_and_stderr(workspace, monkeypatch): assert result["exit_code"] is not None assert result["stderr"] == "pip build detail" assert result["reason"].startswith("ScratchLimit:") + + +@pytest.mark.parametrize("project", [".", "subproject"]) +def test_preparation_uses_project_gradle_wrapper(workspace, monkeypatch, project): + repo = workspace / "checkout" + repo.mkdir() + root = repo / project + root.mkdir(exist_ok=True) + # A real local child proves wrapper execution without system Gradle or network. + wrapper = root / "gradlew" + wrapper.write_text( + 'test "$1" = "--no-daemon" || exit 1\n' + 'test "$2" = "--project-dir" || exit 2\n' + 'test "$4" = "assemble" || exit 3\n' + 'mkdir -p "$3/build"\n' + 'printf compiled > "$3/build/product"\n' + ) + wrapper.chmod(0o644) + app = executor.InProcessExecutor(repo, task_boundary=True) + app.prepared_put_url = "https://files.example/output" + blobs = [] + monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data)) + request = spec() + request["demo"]["build"] = {"recipe": "gradle", "arguments": [project]} + result = app.submit("prepare_verification", request) + assert result["status"] == "ok", result + unpacked = workspace / "unpacked" + verify_task.extract_checkout(blobs[0], unpacked) + assert (unpacked / "checkout" / project / "build/product").read_text() == "compiled" + assert json.loads((unpacked / "spec.json").read_text())["demo"]["build"]["recipe"] == "none" + + +def test_preparation_with_many_host_processes(workspace, monkeypatch): + repo = workspace / "checkout" + repo.mkdir() + (repo / "gradlew").write_text('mkdir -p "$3/build"\nprintf compiled > "$3/build/product"\n') + app = executor.InProcessExecutor(repo, task_boundary=True) + app.prepared_put_url = "https://files.example/output" + monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: None) + request = spec() + request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]} + children = [] + try: + for _ in range(300): + pid = os.fork() + if pid == 0: + time.sleep(60) + os._exit(0) + children.append(pid) + result = app.submit("prepare_verification", request) + assert result["status"] == "ok", result + assert (repo / "build/product").read_text() == "compiled" + finally: + for pid in children: + os.kill(pid, signal.SIGKILL) + for pid in children: + os.waitpid(pid, 0) diff --git a/tests/test_task_images.py b/tests/test_task_images.py new file mode 100644 index 0000000..e59c204 --- /dev/null +++ b/tests/test_task_images.py @@ -0,0 +1,164 @@ +"""Offline image contracts; these do not claim a built image or measured size.""" +import json +import re +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +def read(name): + return (ROOT / name).read_text() + + +def instructions(name): + text = re.sub(r"\\\n\s*", " ", read(name)) + return [line for line in text.splitlines() if line and not line.startswith("#")] + + +def workflow(name): + # BaseLoader avoids YAML 1.1 interpreting GitHub's `on` as a boolean. + return yaml.load(read(f".github/workflows/{name}.yml"), Loader=yaml.BaseLoader) + + +@pytest.mark.parametrize("name,variant", [ + ("plane/Dockerfile.engine-task", "BASE"), + ("plane/Dockerfile.search-task", "BROWSER"), + ("Dockerfile", "BASE"), +]) +def test_digest_required_and_code_last(name, variant): + lines = instructions(name) + assert f"ARG TASK_{variant}_DIGEST" in lines + ref = f"ghcr.io/norandom/ousast-task-base:${{TASK_{variant}_TAG}}@${{TASK_{variant}_DIGEST}}" + assert any(line.startswith(f"FROM {ref}") for line in lines) + code = next(i for i, line in enumerate(lines) if line.startswith("COPY src ")) + for i, line in enumerate(lines): + if "apt-get" in line or "--from=joern-build" in line or "frontend-retention-install.py" in line: + assert i < code + if "pip install" in line: + assert "--no-deps --no-cache-dir ." in line + assert i > code + assert not any("venv /venv" in line for line in lines) + + +def test_shared_ancestry_and_replay_exact_engine(): + base = read("plane/Dockerfile.base") + assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS base" in base + assert "FROM ${TASK_BASE_REF} AS base-browser" in base + common, browser = base.split("FROM ${TASK_BASE_REF} AS base-browser") + assert "chromium" not in common and "chromium" in browser + assert "21-jdk-jammy" in common + assert "COPY src" not in base and 'p["dependencies"]' in base + assert "pip install --no-cache-dir -r /tmp/requirements.txt" in base + engine = read("plane/Dockerfile.engine-task") + assert "ARG ENGINE_REF=engine" in engine and "FROM ${ENGINE_REF} AS replay" in engine + wf = workflow("engine-image") + assert wf["jobs"]["replay"]["needs"] == ["pins", "engine"] + builds = [s["with"] for s in wf["jobs"]["replay"]["steps"] if s.get("uses") == "docker/build-push-action@v6"] + assert "ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }}" in builds[0]["build-args"] + base_wf = workflow("task-base-image") + browser_build = next(s["with"] for s in base_wf["jobs"]["base"]["steps"] if s.get("id") == "browser") + assert "@${{ steps.base.outputs.digest }}" in browser_build["build-args"] + # Remote ancestry and anonymous existence are checked before downstream builds. + assert 'manifests["base-browser"][:len(base)] != base' in read(".github/workflows/engine-image.yml") + + +@pytest.mark.parametrize("name", ["engine-image", "task-base-image"]) +def test_triggers_caches_and_digest_outputs(name): + wf = workflow(name) + assert "workflow_dispatch" in wf["on"] + assert "src/**" not in wf["on"]["push"]["paths"] + if name == "engine-image": + assert wf["on"]["push"]["tags"] == ["v*"] + assert "plane/task-base.digest" in wf["on"]["push"]["paths"] + else: + assert set(wf["on"]["push"]["paths"]) == {"plane/Dockerfile.base", "pyproject.toml", "uv.lock"} + builds = [s["with"] for job in wf["jobs"].values() for s in job["steps"] + if s.get("uses") == "docker/build-push-action@v6"] + assert builds + for build in builds: + assert "type=gha" in build["cache-from"] and "type=gha" in build["cache-to"] + assert build["push"] == "true" + assert "outputs.digest" in read(f".github/workflows/{name}.yml") + + +def test_pin_bootstrap_fails_closed(): + pins = json.loads(read("plane/task-base.digest")) + assert set(pins) >= {"base", "base-browser"} + for key in ("base", "base-browser"): + if pins[key] is None: + assert "UNPUBLISHED" in pins["note"] + else: + assert re.fullmatch(r"ghcr.io/norandom/ousast-task-base:base-[a-z0-9-]+@sha256:[0-9a-f]{64}", pins[key]) + wf = read(".github/workflows/engine-image.yml") + assert "raise SystemExit" in wf and '"inspect", "--raw", ref' in wf + assert "needs: pins" in wf + + +@pytest.mark.parametrize("case", ["unpublished", "missing", "wrong-ancestry", "valid"]) +def test_ci_pin_gate_offline(case, tmp_path, monkeypatch): + """Exercise the actual workflow Python, with only registry reads replaced.""" + import subprocess + + pin_step = next(s for s in workflow("engine-image")["jobs"]["pins"]["steps"] if s.get("id") == "pins") + script = pin_step["run"].split("<<'PYTHON'\n", 1)[1].rsplit("PYTHON", 1)[0] + base = "ghcr.io/norandom/ousast-task-base:base-1234567@sha256:" + "a" * 64 + browser = "ghcr.io/norandom/ousast-task-base:base-1234567-browser@sha256:" + "b" * 64 + (tmp_path / "plane").mkdir() + (tmp_path / "plane/task-base.digest").write_text(json.dumps({ + "base": None if case == "unpublished" else base, "base-browser": browser, + })) + output = tmp_path / "output" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + calls = [] + + def inspect(argv): + calls.append(argv) + assert argv[:5] == ["docker", "buildx", "imagetools", "inspect", "--raw"] + if case == "missing": + raise subprocess.CalledProcessError(1, argv) + layers = [{"digest": "sha256:shared"}] + if argv[-1] == browser: + if case == "wrong-ancestry": + layers = [{"digest": "sha256:different"}] + layers.append({"digest": "sha256:chromium"}) + return json.dumps({"layers": layers}).encode() + + monkeypatch.setattr(subprocess, "check_output", inspect) + if case == "valid": + exec(compile(script, "workflow-pin-gate", "exec"), {}) + assert "base_digest=sha256:" in output.read_text() + assert "browser_digest=sha256:" in output.read_text() + assert len(calls) == 2 + else: + expected = subprocess.CalledProcessError if case == "missing" else SystemExit + with pytest.raises(expected): + exec(compile(script, "workflow-pin-gate", "exec"), {}) + if case == "unpublished": + assert not calls + + +def test_base_preserves_executor_build_toolchains(): + common = read("plane/Dockerfile.base").split("FROM ${TASK_BASE_REF} AS base-browser")[0] + commands = " ".join(instructions("plane/Dockerfile.base")).split("FROM ${TASK_BASE_REF} AS base-browser")[0] + install = re.search(r"apt-get install -y --no-install-recommends\s+(.*?)\s+&&", commands).group(1) + packages = set(install.split()) + assert packages == { + "nodejs", "npm", "node-typescript", "php-cli", "composer", "maven", + "sqlite3", "bubblewrap", "util-linux", "git", "ca-certificates", + "libstdc++6", "zlib1g", + } + assert "gradle" not in packages + assert "FROM eclipse-temurin:21-jdk-jammy AS jdk" in common + assert "COPY --from=jdk /opt/java/openjdk /opt/java/openjdk" in common + for command in ("javac -version", "npm --version", "composer --version", "mvn --version", + "python3 -m venv /venv", "/venv/bin/pip install"): + assert command in common + assert "joern" not in common.lower() + assert "apt-get clean" in common + for directory in ("doc", "man", "info"): + assert f"path-exclude=/usr/share/{directory}/*" in common + assert f" /usr/share/{directory}/*" in common