merge(image-layering): shared task base and code-last layering [skip ci]

Co-Authored-By: Claude Fable 5.1 <[email protected]>

# Conflicts:
#	tests/test_search_task_storage.py
This commit is contained in:
Marius Ciepluch
2026-10-05 20:44:02 +02:00
12 changed files with 670 additions and 141 deletions
+144 -17
View File
@@ -3,18 +3,11 @@ on:
workflow_dispatch:
push:
branches: [main]
tags: ['v*']
paths:
- 'src/**'
- 'benchmarks/learn/engine_trace*.py'
- 'benchmarks/learn/engine_task_entry.py'
- 'benchmarks/unseen/replay_entry.py'
- 'benchmarks/learn/engine_worker_service.py'
- 'plane/Dockerfile.*'
- 'plane/task-base.digest'
- 'Dockerfile'
- 'plane/Dockerfile.engine-task'
- 'plane/Dockerfile.search-task'
- 'ops/frontend-retention/install.py'
- 'pyproject.toml'
- '.github/workflows/engine-image.yml'
permissions:
contents: read
packages: write
@@ -22,7 +15,40 @@ concurrency:
group: engine-image-${{ github.ref }}
cancel-in-progress: false
jobs:
pins:
runs-on: ubuntu-latest
outputs:
base_tag: ${{ steps.pins.outputs.base_tag }}
base_digest: ${{ steps.pins.outputs.base_digest }}
browser_tag: ${{ steps.pins.outputs.browser_tag }}
browser_digest: ${{ steps.pins.outputs.browser_digest }}
steps:
- uses: actions/checkout@v5
- uses: docker/setup-buildx-action@v3
# No login: every pinned base must be anonymously pullable.
- id: pins
run: |
python3 - <<'PYTHON'
import json, os, re, subprocess
from pathlib import Path
pins = json.loads(Path("plane/task-base.digest").read_text())
manifests = {}
for key, output in (("base", "base"), ("base-browser", "browser")):
ref = pins.get(key)
pattern = r"ghcr.io/norandom/ousast-task-base:(base-[a-z0-9-]+)@(sha256:[0-9a-f]{64})"
match = re.fullmatch(pattern, ref or "")
if not match:
raise SystemExit(f"{key}: unpublished/invalid pin; run task-base-image and update plane/task-base.digest")
raw = subprocess.check_output(["docker", "buildx", "imagetools", "inspect", "--raw", ref])
manifests[key] = json.loads(raw)["layers"]
with open(os.environ["GITHUB_OUTPUT"], "a") as out:
out.write(f"{output}_tag={match[1]}\n{output}_digest={match[2]}\n")
base = manifests["base"]
if not base or manifests["base-browser"][:len(base)] != base:
raise SystemExit("Browser pin does not share the pinned base's layers")
PYTHON
image:
needs: pins
runs-on: ubuntu-latest
strategy:
fail-fast: false
@@ -30,12 +56,6 @@ jobs:
include:
- package: openultrasast
dockerfile: Dockerfile
- package: ousast-engine-task
dockerfile: plane/Dockerfile.engine-task
target: engine
- package: ousast-replay-task
dockerfile: plane/Dockerfile.engine-task
target: replay
- package: ousast-search-task
dockerfile: plane/Dockerfile.search-task
steps:
@@ -58,9 +78,18 @@ jobs:
with:
context: .
file: ${{ matrix.dockerfile }}
target: ${{ matrix.target }}
platforms: linux/amd64
push: true
provenance: false
build-args: |
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
cache-from: |
type=gha,scope=${{ matrix.package }}
type=gha,scope=ousast-engine-task
cache-to: type=gha,mode=max,scope=${{ matrix.package }}
tags: |
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
${{ steps.names.outputs.image }}:main
@@ -70,3 +99,101 @@ jobs:
run: |
echo "$IMAGE@$DIGEST"
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
docker logout ghcr.io
docker buildx imagetools inspect "$IMAGE@$DIGEST"
engine:
needs: pins
runs-on: ubuntu-latest
outputs:
image: ${{ steps.names.outputs.image }}
digest: ${{ steps.build.outputs.digest }}
steps:
- uses: actions/checkout@v5
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- id: names
env:
OWNER: ${{ github.repository_owner }}
PACKAGE: ousast-engine-task
run: |
echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT"
echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
- id: build
uses: docker/build-push-action@v6
with:
context: .
file: plane/Dockerfile.engine-task
target: engine
platforms: linux/amd64
push: true
provenance: false
build-args: |
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
cache-from: type=gha,scope=ousast-engine-task
cache-to: type=gha,mode=max,scope=ousast-engine-task
tags: |
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
${{ steps.names.outputs.image }}:main
- env:
IMAGE: ${{ steps.names.outputs.image }}
DIGEST: ${{ steps.build.outputs.digest }}
run: |
echo "$IMAGE@$DIGEST"
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
docker logout ghcr.io
docker buildx imagetools inspect "$IMAGE@$DIGEST"
replay:
needs: [pins, engine]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- id: names
env:
OWNER: ${{ github.repository_owner }}
PACKAGE: ousast-replay-task
run: |
echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT"
echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
- id: build
uses: docker/build-push-action@v6
with:
context: .
file: plane/Dockerfile.engine-task
target: replay
platforms: linux/amd64
push: true
provenance: false
build-args: |
ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }}
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
cache-from: |
type=gha,scope=ousast-replay-task
type=gha,scope=ousast-engine-task
cache-to: type=gha,mode=max,scope=ousast-replay-task
tags: |
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
${{ steps.names.outputs.image }}:main
- env:
IMAGE: ${{ steps.names.outputs.image }}
DIGEST: ${{ steps.build.outputs.digest }}
run: |
echo "$IMAGE@$DIGEST"
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
docker logout ghcr.io
docker buildx imagetools inspect "$IMAGE@$DIGEST"
+78
View File
@@ -0,0 +1,78 @@
name: Task base image
on:
workflow_dispatch:
push:
branches: [main]
paths:
- 'plane/Dockerfile.base'
- 'pyproject.toml'
- 'uv.lock'
permissions:
contents: read
packages: write
concurrency:
group: task-base-image
cancel-in-progress: false
jobs:
base:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- id: names
run: |
echo "tag=base-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
content=$(cat plane/Dockerfile.base pyproject.toml uv.lock | sha256sum | cut -d' ' -f1)
echo "content=base-content-$content" >> "$GITHUB_OUTPUT"
- id: base
uses: docker/build-push-action@v6
with:
context: .
file: plane/Dockerfile.base
target: base
platforms: linux/amd64
push: true
provenance: false
sbom: false
tags: |
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }}
cache-from: type=gha,scope=task-base
cache-to: type=gha,mode=max,scope=task-base
- id: browser
uses: docker/build-push-action@v6
with:
context: .
file: plane/Dockerfile.base
target: base-browser
platforms: linux/amd64
push: true
provenance: false
sbom: false
build-args: |
TASK_BASE_REF=ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }}
tags: |
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }}-browser
cache-from: type=gha,scope=task-base-browser
cache-to: type=gha,mode=max,scope=task-base-browser
- name: Print manual pin update and verify public access
env:
BASE: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }}
BROWSER: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser@${{ steps.browser.outputs.digest }}
run: |
python3 - <<'PYTHON' >> "$GITHUB_STEP_SUMMARY"
import json, os
print("Replace plane/task-base.digest with this JSON after both public checks pass:")
print("```json")
print(json.dumps({"base": os.environ["BASE"], "base-browser": os.environ["BROWSER"]}, indent=2))
print("```")
PYTHON
docker logout ghcr.io
docker buildx imagetools inspect "$BASE"
docker buildx imagetools inspect "$BROWSER"
+19 -42
View File
@@ -1,26 +1,14 @@
# OpenUltraSAST with its CPG engine, in one image.
#
# The model layer's arbiter is Joern, and `openultrasast.cpg.backend` reaches it by subprocess. Shipping both
# in one image is what makes that simple: Joern is on PATH inside, so there is no container-to-container call,
# no docker socket, no path translation between host and container, and no second code path in the backend to
# get wrong. The alternative -- tool and engine as sibling services -- needs the tool to exec into the other
# container, which means mounting the docker socket, and that is a large amount of privilege for a linter.
#
# The image is big (a JRE plus ~2 GB of Joern). That is the honest cost of bundling an engine, and it buys a
# contributor a working tool from `docker compose run` with nothing installed on their machine.
FROM eclipse-temurin:21-jre-noble
LABEL org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
# syntax=docker/dockerfile:1
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
ARG TASK_BASE_TAG=base-unpublished
ARG TASK_BASE_DIGEST
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS joern-build
ENV PATH="/opt/joern-cli:${PATH}"
ARG JOERN_VERSION=v4.0.625
ARG ARCHIVE=joern-cli-linux-x86_64.zip
# php-cli is required by php2cpg, which drives PHP-Parser and shells out to a real interpreter. Without it a
# PHP tree fails with an opaque "Process exited with code 1" -- measured on this project, not assumed.
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
python3 python3-venv python3-pip unzip curl ca-certificates git php-cli \
&& apt-get install -y --no-install-recommends curl unzip \
&& rm -rf /var/lib/apt/lists/*
RUN set -eux; \
@@ -32,39 +20,28 @@ RUN set -eux; \
unzip -q "${ARCHIVE}" -d /opt; \
rm -f "${ARCHIVE}" "${ARCHIVE}.sha512" checksum.sha512; \
/opt/joern-cli/joern-parse --help > /dev/null
ENV PATH="/opt/joern-cli:${PATH}"
# This release ships the JS launcher with a .sh suffix; the backend's direct fallback uses
# the same extensionless command convention as the other frontends.
RUN if [ ! -e /opt/joern-cli/jssrc2cpg ] && [ -x /opt/joern-cli/jssrc2cpg.sh ]; then \
ln -s jssrc2cpg.sh /opt/joern-cli/jssrc2cpg; \
fi
COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py
RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \
&& rm /tmp/frontend-retention-install.py
WORKDIR /app
COPY pyproject.toml README.md ./
COPY src ./src
COPY benchmarks ./benchmarks
# The core install is zero-dependency by design; `semantic` adds the tree-sitter grammars the candidate
# enumerator uses. The LLM endpoint stays optional and is configured by environment, never baked in.
RUN python3 -m venv /venv \
&& /venv/bin/pip install --no-cache-dir -e ".[semantic,s3]"
ENV PATH="/venv/bin:${PATH}"
# Analysis never needs root.
# Noble may already provide the ubuntu account at UID 1000. Reuse that identity rather than
# failing installation or creating a second account with the same UID.
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS engine
ENV PATH="/opt/joern-cli:${PATH}"
COPY --from=joern-build /opt/joern-cli /opt/joern-cli
# Create runtime identity before the application layers.
RUN if getent passwd 1000 > /dev/null; then \
usermod --login ousast --home /home/ousast --move-home "$(getent passwd 1000 | cut -d: -f1)"; \
else \
useradd --create-home --uid 1000 ousast; \
fi \
&& chown -R ousast /app
else useradd --create-home --uid 1000 ousast; fi
WORKDIR /app
COPY pyproject.toml README.md LICENSE ./
COPY src ./src
RUN --mount=type=cache,target=/root/.cache/pip \
/venv/bin/pip install --no-deps --no-cache-dir .
COPY --chown=1000 benchmarks ./benchmarks
RUN chown -R ousast /app
USER ousast
ENTRYPOINT ["python", "-m", "openultrasast.cli"]
CMD ["--help"]
+99
View File
@@ -260,3 +260,102 @@ filesystem quota; programs that hardcode `/tmp` still consume the worker's RAM.
Repository clones must use HTTPS `git clone https://github.com/...git` or codeload
archives, never `api.github.com`: tasks share one public IP. Search task inputs are
presigned checkout archives, so these entrypoints do not clone repositories themselves.
## Image layers, publication and node cache budget
The task family has one shared runtime base, `ousast-task-base`, built from the
public AX runner v0.3.1. `base-browser` adds Chromium to that exact published base
(by digest); engine and replay do not carry Chromium. The VM/kind `openultrasast`
image uses the same base and Joern recipe. Replay inherits the published engine
digest in CI, so even a cold build cannot accidentally duplicate its Joern layers.
Java is **Temurin 21 JDK**, copied from `eclipse-temurin:21-jdk-jammy`.
`ops/frontend-retention/install.py` explicitly requires Java 21 and invokes
`java -cp ... dotty.tools.dotc.Main` using Joern's bundled Scala compiler.
The shared JDK also provides `javac` for Maven/Gradle project builds.
Curl/unzip and retention build scratch stay in a discarded Joern build stage.
Joern stays above the base in the engine image (also used by replay and the VM
lane); its compiler jars and retention provenance must not be blindly pruned.
Runtime dependencies from `pyproject.toml` (core plus semantic/s3 for the VM lane)
precede project code. Package installation uses `--no-deps --no-cache-dir`;
its isolated packaging backend is temporary.
The base retains upstream Python 3.12 with venv/pip. Its explicit apt package list
is `nodejs npm node-typescript php-cli composer maven sqlite3 bubblewrap util-linux
git ca-certificates libstdc++6 zlib1g`. Composer is the trixie apt package.
Chromium is installed only in `base-browser`. Apt caches/lists and docs/man/info
are cleaned; dpkg excludes new docs/man/info. Files already in inherited AX
layers still occupy space even if deleted.
Executor preparation supports pip, npm, Composer, Maven and Gradle builds.
There is **no installed Gradle binary**: the executor invokes the selected project's
`gradlew` through `/bin/sh`, including for wrappers without executable permissions.
Gradle-wrapper distribution downloads and dependency downloads go through
**public HTTPS from the executor**, subject to its egress policy and deadline.
The verifier receives prepared products and performs no dependency downloads.
Rebuild policy:
- Dependency/toolchain changes: run **Task base image** (also triggered by changes
to `plane/Dockerfile.base`, `pyproject.toml`, or `uv.lock`). It publishes
`base-<short sha>` and `base-content-<input hash>` tags, plus their `-browser`
variants, and prints immutable digest references. Content tags identify build
inputs; they do not claim reproducible upstream apt/PyPI resolution. The lockfile
triggers refreshes but pip installs the declared runtime constraints, not a uv
frozen environment. Deliberate upstream refreshes use manual dispatch.
- After both targets publish, ensure `ousast-task-base` is **public** in GHCR's
package settings. Copy the workflow summary's JSON into `plane/task-base.digest`
and submit that pin update for normal review. CI does not commit on your behalf.
The initial null pins mean **unpublished**, not a usable bootstrap image.
- A pin update or `plane/Dockerfile.*`/root Dockerfile change on main runs **Engine
image**. For source, entrypoint or retention-installer changes, dispatch it once
for the desired revision, or push a `v*` release tag. Source-only merges no longer
publish images automatically. Never rebuild per task/run. All builds use scoped
GHA caches; pin checks precede builds and require anonymous access plus identical
base-layer ancestry for the browser variant.
- Final package names must also be public in GHCR. Publication checks anonymous
access after logout and fails if the package is private. Set visibility on first
publication and rerun as needed. Deploy only the reported `image@sha256:...`
references in the cluster profile/Task manifests, never `main` or a release tag.
Publication does not automatically change deployed Task pins.
For a local task build, pass `TASK_BASE_TAG` and `TASK_BASE_DIGEST` from the base
pin; search instead takes `TASK_BROWSER_TAG` and `TASK_BROWSER_DIGEST`. Replay's
`ENGINE_REF` defaults to its local engine stage; set it to the published engine
reference to reuse exactly those layers. When building `base-browser` locally,
pass `TASK_BASE_REF=<full base tag@digest>` for the same ancestry as CI.
The operator's capacity rule is **all images in rotation, unpacked, together
at most approximately 9 GiB**, including rollback revisions. Working builds take
priority over minimizing an individual executor image; its previous 2.13 GB
unpacked (~1.98 GiB) was acceptable. Retire unused Task templates and image
revisions before rotating a large base/Joern generation. Shared layers occupy
storage once where the runtime deduplicates them, but do not assume that turns
a sum above 9 GiB into an acceptable rotation. Record both the sum of unpacked
image sizes and actual unique-layer storage. Never prune images used by live tasks.
Expected footprint (planning estimates only; no images built for this change):
| Image | Expected unpacked size | Basis / uncertainty |
| --- | --- | --- |
| Shared base | about 1.3–1.9 GiB | ~454 MB inherited AX layers + ~281 MB JDK baseline + npm/Composer/Maven, language runtimes and venv/extras; apt dependency closure unmeasured |
| Browser base | about 1.65–2.45 GiB | Shared base + roughly 350–550 MB Chromium/dependencies |
| Search executor/verifier | about 1.7–2.5 GiB | Browser base + application; previous 2.13 GB unpacked is a baseline, not a ceiling |
| Engine | roughly 3.5–6 GiB | Restored shared base + Joern; the supplied 2.16 GB compressed engine cannot establish its unpacked size |
| Replay | engine + a few KiB | Same engine digest plus replay entrypoint |
| VM/kind openultrasast | roughly 3.5–6 GiB plus benchmark assets | Same base/Joern, runtime extras shared, benchmark payload retained |
An engine + search rotation therefore sums to about **5.2–8.5 GiB** before
rollback images. Adding replay as another unpacked image brings the sum to about
**8.7–14.5 GiB**; this rotation is not demonstrated to fit 9 GiB. Engine/replay
share almost all layers, but the operator must confirm the accounting and measured
footprint before selecting a rotation. Extra revisions and the VM image add to
the budget if retained there.
The previous search history included a 1.37 GB apt layer and 281 MB JDK; no
per-package apt breakdown is available. Removing system Gradle and docs/caches
may save space, while retained build tools and shared Python extras cost space.
These ranges are not measured savings or a capacity guarantee. After publication,
measure unpacked images and unique layers for the entire intended rotation, and
verify build recipes, PHP/JavaScript retention, runner startup and Chromium before
rollout. There is no separate <1.5 GiB executor gate.
+41
View File
@@ -0,0 +1,41 @@
# syntax=docker/dockerfile:1
# base-browser is built against the published base digest in CI; local builds
# default to the base stage. Never independently rebuild its shared ancestry.
ARG TASK_BASE_REF=base
FROM eclipse-temurin:21-jdk-jammy AS jdk
FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS base
LABEL org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
USER root
COPY --from=jdk /opt/java/openjdk /opt/java/openjdk
ENV JAVA_HOME=/opt/java/openjdk
ENV PATH="/venv/bin:/opt/java/openjdk/bin:${PATH}"
# Use upstream /usr/local Python 3.12 (also required by frontend retention).
# Its venv/pip and the toolchains below support every executor build recipe.
# Gradle projects use their wrapper; no system Gradle is installed.
RUN printf 'path-exclude=/usr/share/doc/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/info/*\n' \
> /etc/dpkg/dpkg.cfg.d/01-no-docs \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
nodejs npm node-typescript php-cli composer maven sqlite3 bubblewrap util-linux git ca-certificates libstdc++6 zlib1g \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /usr/share/doc/* /usr/share/man/* /usr/share/info/* \
/opt/java/openjdk/lib/src.zip /opt/java/openjdk/man /root/.cache
# Metadata only: install runtime dependencies, including the VM lane's extras.
# No project package or packaging backend enters this shared environment.
COPY pyproject.toml /tmp/dependencies/pyproject.toml
RUN python3 -c 'import pathlib,tomllib; p=tomllib.loads(pathlib.Path("/tmp/dependencies/pyproject.toml").read_text())["project"]; print("\n".join(p["dependencies"] + p["optional-dependencies"]["semantic"] + p["optional-dependencies"]["s3"]))' > /tmp/requirements.txt \
&& python3 -m venv /venv \
&& /venv/bin/pip install --no-cache-dir -r /tmp/requirements.txt \
&& rm -rf /tmp/dependencies /tmp/requirements.txt /root/.cache \
&& python3 --version && java -version && javac -version && node --version && npm --version && tsc --version \
&& php --version && composer --version && mvn --version && sqlite3 --version && bwrap --version \
&& setpriv --version && prlimit --version && nsenter --version
FROM ${TASK_BASE_REF} AS base-browser
RUN apt-get update \
&& apt-get install -y --no-install-recommends chromium \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /usr/share/doc/* /usr/share/man/* /usr/share/info/* /root/.cache \
&& chromium --version
+17 -29
View File
@@ -1,23 +1,15 @@
# Build from the repository root. AX v0.3.1 starts spec.command immediately;
# preserve its runner as PID 1 and put the engine command in the Task manifest.
FROM eclipse-temurin:21-jre-jammy AS jre
FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS engine
LABEL org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
USER root
# syntax=docker/dockerfile:1
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
ARG TASK_BASE_TAG=base-unpublished
ARG TASK_BASE_DIGEST
ARG ENGINE_REF=engine
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS joern-build
ENV PATH="/opt/joern-cli:${PATH}"
ARG JOERN_VERSION=v4.0.625
ARG ARCHIVE=joern-cli-linux-x86_64.zip
# The AX runner is Python 3.12 / Debian slim. Copy the Java 21 runtime from
# Jammy (older glibc than Debian bookworm) instead of relying on distro JRE versions.
COPY --from=jre /opt/java/openjdk /opt/java/openjdk
ENV JAVA_HOME=/opt/java/openjdk
ENV PATH="/venv/bin:/opt/java/openjdk/bin:/opt/joern-cli:${PATH}"
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
php-cli unzip curl ca-certificates libstdc++6 zlib1g \
&& apt-get install -y --no-install-recommends curl unzip \
&& rm -rf /var/lib/apt/lists/*
RUN set -eux; \
@@ -37,24 +29,20 @@ COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py
RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \
&& rm /tmp/frontend-retention-install.py
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS engine
ENV PATH="/opt/joern-cli:${PATH}"
COPY --from=joern-build /opt/joern-cli /opt/joern-cli
WORKDIR /app
COPY pyproject.toml README.md LICENSE ./
COPY src ./src
RUN --mount=type=cache,target=/root/.cache/pip \
/venv/bin/pip install --no-deps --no-cache-dir .
COPY benchmarks/learn/engine_task_entry.py \
benchmarks/learn/engine_trace_worker.py \
benchmarks/learn/engine_trace_parse.py ./benchmarks/learn/
# Worker imports are standard-library plus engine modules. Isolate them from
# the AX runner's Python environment and omit PyYAML, boto3 and all extras.
RUN python3 -m venv /venv \
&& /venv/bin/pip install --no-cache-dir --no-deps . \
&& /venv/bin/python -I -c 'import pathlib, runpy, sys; sys.modules.update(yaml=None, boto3=None); sys.path.insert(0, "/app/benchmarks/learn"); p = pathlib.Path("/app/benchmarks/learn/engine_trace_worker.py"); print("Worker import smoke bytes:", len(p.read_bytes())); runpy.run_path(str(p), run_name="engine_worker_import_smoke")'
RUN /venv/bin/python -I -c 'import pathlib, runpy, sys; sys.modules.update(yaml=None, boto3=None); sys.path.insert(0, "/app/benchmarks/learn"); p = pathlib.Path("/app/benchmarks/learn/engine_trace_worker.py"); print("Worker import smoke bytes:", len(p.read_bytes())); runpy.run_path(str(p), run_name="engine_worker_import_smoke")'
# ENTRYPOINT and CMD belong to the upstream AX runner.
# Do not set ENTRYPOINT or CMD: both belong to the upstream AX runner.
FROM engine AS replay
RUN apt-get update \
&& apt-get install -y --no-install-recommends git \
&& rm -rf /var/lib/apt/lists/*
FROM ${ENGINE_REF} AS replay
COPY benchmarks/unseen/replay_entry.py ./benchmarks/unseen/replay_entry.py
RUN /venv/bin/pip install --no-cache-dir . \
&& /venv/bin/ousast pre-push --help > /dev/null
RUN /venv/bin/ousast pre-push --help > /dev/null
+13 -29
View File
@@ -1,35 +1,19 @@
# Build from the repository root; retain the AX runner entrypoint and command.
FROM eclipse-temurin:17-jdk-jammy AS jdk
FROM ghcr.io/norandom/ax-task-runner:v0.3.1
LABEL org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
# syntax=docker/dockerfile:1
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
ARG TASK_BROWSER_TAG=base-unpublished-browser
ARG TASK_BROWSER_DIGEST
FROM ghcr.io/norandom/ousast-task-base:${TASK_BROWSER_TAG}@${TASK_BROWSER_DIGEST}
USER root
COPY --from=jdk /opt/java/openjdk /opt/java/openjdk
ENV JAVA_HOME=/opt/java/openjdk
ENV PATH="/venv/bin:/opt/java/openjdk/bin:${PATH}"
# dpkg exclusions prevent docs/manpages entering layers, including transitive deps.
RUN printf 'path-exclude=/usr/share/doc/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/info/*\n' \
> /etc/dpkg/dpkg.cfg.d/01-no-docs \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
python3 python3-venv python3-pip nodejs npm node-typescript php-cli composer maven gradle \
sqlite3 bubblewrap util-linux chromium git ca-certificates \
&& rm -rf /var/lib/apt/lists/* /opt/java/openjdk/lib/src.zip /opt/java/openjdk/man
WORKDIR /app
COPY pyproject.toml README.md LICENSE ./
COPY src ./src
# Includes the verifier-owned recipes, oracles, and key-free executor entrypoints.
RUN python3 -m venv /venv \
&& /venv/bin/pip install --no-cache-dir . \
&& python3 --version && node --version && tsc --version && php --version \
&& javac -version && sqlite3 --version && bwrap --version \
&& setpriv --version && prlimit --version && nsenter --version && chromium --version
# Dedicated identity for artefacts, apps and Chromium; the verifier keeps root.
RUN groupadd --gid 10001 search \
&& useradd --create-home --uid 10001 --gid 10001 search
# gVisor is the outer boundary; retain inner bwrap when its capability probe succeeds.
# The Python base image sets GPG_KEY to the public CPython release-signing fingerprint (not a secret).
# Clear it so the executor's guard against inherited credentials stays strict and still passes.
# Upstream's public CPython signing fingerprint trips the credential guard.
ENV GPG_KEY=
WORKDIR /app
COPY pyproject.toml README.md LICENSE ./
COPY src ./src
RUN --mount=type=cache,target=/root/.cache/pip \
/venv/bin/pip install --no-deps --no-cache-dir .
# Entrypoints and verifier-owned recipes/oracles are included in src.
USER root
# Do not replace the AX runner's ENTRYPOINT or CMD.
# Preserve the AX runner ENTRYPOINT and CMD.
+5
View File
@@ -0,0 +1,5 @@
{
"base": null,
"base-browser": null,
"note": "UNPUBLISHED: run task-base-image, then replace with the two full tag@sha256 references from its summary. Never invent digests."
}
+27 -2
View File
@@ -274,9 +274,31 @@ class InProcessExecutor:
}
def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]:
process_limit = None
try:
uid = os.getuid()
current = 0
for entry in Path("/proc").iterdir():
if not entry.name.isdecimal():
continue
try:
status = (entry / "status").read_text()
except FileNotFoundError:
continue
fields = dict(line.split(":", 1) for line in status.splitlines() if ":" in line)
# The kernel's NPROC counter counts threads, not processes: a single IDE or agent
# process with 60 threads uses 60 of the allowance.
if int(fields["Uid"].split()[0]) == uid:
current += int(fields.get("Threads", "1").strip() or 1)
process_limit = current + 256
except (OSError, ValueError, IndexError, StopIteration):
pass
def constrain() -> None:
resource.setrlimit(resource.RLIMIT_FSIZE, (limits.get("disk_bytes", task_storage.DEFAULT_SCRATCH_BYTES),) * 2)
resource.setrlimit(resource.RLIMIT_NPROC, (128, 128))
# NPROC counts all processes of the real uid, so an absolute 128 blocks builds on busy hosts.
if process_limit is not None:
resource.setrlimit(resource.RLIMIT_NPROC, (process_limit,) * 2)
resource.setrlimit(resource.RLIMIT_AS, (limits.get("memory_bytes", 256 * 1024**2),) * 2)
resource.setrlimit(resource.RLIMIT_CPU, (math.ceil(timeout) + 1,) * 2)
@@ -391,7 +413,10 @@ class InProcessExecutor:
if recipe["recipe"] == "maven":
argv += ["package", "-DskipTests"]
elif recipe["recipe"] == "gradle":
argv += ["assemble"]
# The project pins its Gradle version through its wrapper.
# Invoke with sh because checkout archives may lose execute bits.
wrapper = self._path(str(path.relative_to(self.repo) / "gradlew"))
argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"]
built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES})
if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
return {**built, "status": "could_not_build"}
+8 -8
View File
@@ -4,17 +4,17 @@ import yaml
def test_image_matches_runner_and_publishes_digest():
engine = Path("plane/Dockerfile.engine-task").read_text()
base = Path("plane/Dockerfile.base").read_text()
image = Path("plane/Dockerfile.search-task").read_text()
runner = "FROM ghcr.io/norandom/ax-task-runner:v0.3.1"
assert runner in engine and runner in image
for package in ("python3", "nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"):
assert package in image
assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1" in base
for package in ("nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"):
assert package in base
assert "python3 -m venv /venv" in base
assert image.rsplit("USER ", 1)[1].startswith("root\n")
assert "--uid 10001 --gid 10001 search" in image
assert "setpriv --version" in image
assert "17-jdk" in image and "--no-install-recommends" in image
assert "rm -rf /var/lib/apt/lists/" in image
assert "setpriv --version" in base
assert "21-jdk" in base and "--no-install-recommends" in base
assert "rm -rf /var/lib/apt/lists/" in base
workflow_text = Path(".github/workflows/engine-image.yml").read_text()
entries = yaml.safe_load(workflow_text)["jobs"]["image"]["strategy"]["matrix"]["include"]
assert {"package": "ousast-search-task", "dockerfile": "plane/Dockerfile.search-task"} in entries
+55 -14
View File
@@ -2,7 +2,9 @@
import json
import os
import signal
import sys
import time
from pathlib import Path
import pytest
@@ -224,19 +226,58 @@ def test_scratch_failure_preserves_exit_and_stderr(workspace, monkeypatch):
assert result["reason"].startswith("ScratchLimit:")
def test_capacity_deadline_plain_diagnostic_survives():
from benchmarks.ax.batch import CapacityDeadline
message = "no free workers available before deadline"
assert task_storage.diagnostic(message) == message
assert task_storage.exception_reason(CapacityDeadline(message)) == "CapacityDeadline: " + message
@pytest.mark.parametrize("project", [".", "subproject"])
def test_preparation_uses_project_gradle_wrapper(workspace, monkeypatch, project):
repo = workspace / "checkout"
repo.mkdir()
root = repo / project
root.mkdir(exist_ok=True)
# A real local child proves wrapper execution without system Gradle or network.
wrapper = root / "gradlew"
wrapper.write_text(
'test "$1" = "--no-daemon" || exit 1\n'
'test "$2" = "--project-dir" || exit 2\n'
'test "$4" = "assemble" || exit 3\n'
'mkdir -p "$3/build"\n'
'printf compiled > "$3/build/product"\n'
)
wrapper.chmod(0o644)
app = executor.InProcessExecutor(repo, task_boundary=True)
app.prepared_put_url = "https://files.example/output"
blobs = []
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
request = spec()
request["demo"]["build"] = {"recipe": "gradle", "arguments": [project]}
result = app.submit("prepare_verification", request)
assert result["status"] == "ok", result
unpacked = workspace / "unpacked"
verify_task.extract_checkout(blobs[0], unpacked)
assert (unpacked / "checkout" / project / "build/product").read_text() == "compiled"
assert json.loads((unpacked / "spec.json").read_text())["demo"]["build"]["recipe"] == "none"
def test_cleanup_diagnostic_retains_both_causes():
error = ValueError("original apply failed")
error.executor_cleanup_reason = "executor cleanup failed: " + "details " * 100 + "delete failed"
error.args = (str(error) + "; " + error.executor_cleanup_reason,)
reason = task_storage.exception_reason(error)
assert reason.startswith("ValueError: original apply failed;")
assert reason.endswith("delete failed")
assert len(reason) <= 300
def test_preparation_with_many_host_processes(workspace, monkeypatch):
repo = workspace / "checkout"
repo.mkdir()
(repo / "gradlew").write_text('mkdir -p "$3/build"\nprintf compiled > "$3/build/product"\n')
app = executor.InProcessExecutor(repo, task_boundary=True)
app.prepared_put_url = "https://files.example/output"
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: None)
request = spec()
request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]}
children = []
try:
for _ in range(300):
pid = os.fork()
if pid == 0:
time.sleep(60)
os._exit(0)
children.append(pid)
result = app.submit("prepare_verification", request)
assert result["status"] == "ok", result
assert (repo / "build/product").read_text() == "compiled"
finally:
for pid in children:
os.kill(pid, signal.SIGKILL)
for pid in children:
os.waitpid(pid, 0)
+164
View File
@@ -0,0 +1,164 @@
"""Offline image contracts; these do not claim a built image or measured size."""
import json
import re
from pathlib import Path
import pytest
import yaml
ROOT = Path(__file__).resolve().parents[1]
def read(name):
return (ROOT / name).read_text()
def instructions(name):
text = re.sub(r"\\\n\s*", " ", read(name))
return [line for line in text.splitlines() if line and not line.startswith("#")]
def workflow(name):
# BaseLoader avoids YAML 1.1 interpreting GitHub's `on` as a boolean.
return yaml.load(read(f".github/workflows/{name}.yml"), Loader=yaml.BaseLoader)
@pytest.mark.parametrize("name,variant", [
("plane/Dockerfile.engine-task", "BASE"),
("plane/Dockerfile.search-task", "BROWSER"),
("Dockerfile", "BASE"),
])
def test_digest_required_and_code_last(name, variant):
lines = instructions(name)
assert f"ARG TASK_{variant}_DIGEST" in lines
ref = f"ghcr.io/norandom/ousast-task-base:${{TASK_{variant}_TAG}}@${{TASK_{variant}_DIGEST}}"
assert any(line.startswith(f"FROM {ref}") for line in lines)
code = next(i for i, line in enumerate(lines) if line.startswith("COPY src "))
for i, line in enumerate(lines):
if "apt-get" in line or "--from=joern-build" in line or "frontend-retention-install.py" in line:
assert i < code
if "pip install" in line:
assert "--no-deps --no-cache-dir ." in line
assert i > code
assert not any("venv /venv" in line for line in lines)
def test_shared_ancestry_and_replay_exact_engine():
base = read("plane/Dockerfile.base")
assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS base" in base
assert "FROM ${TASK_BASE_REF} AS base-browser" in base
common, browser = base.split("FROM ${TASK_BASE_REF} AS base-browser")
assert "chromium" not in common and "chromium" in browser
assert "21-jdk-jammy" in common
assert "COPY src" not in base and 'p["dependencies"]' in base
assert "pip install --no-cache-dir -r /tmp/requirements.txt" in base
engine = read("plane/Dockerfile.engine-task")
assert "ARG ENGINE_REF=engine" in engine and "FROM ${ENGINE_REF} AS replay" in engine
wf = workflow("engine-image")
assert wf["jobs"]["replay"]["needs"] == ["pins", "engine"]
builds = [s["with"] for s in wf["jobs"]["replay"]["steps"] if s.get("uses") == "docker/build-push-action@v6"]
assert "ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }}" in builds[0]["build-args"]
base_wf = workflow("task-base-image")
browser_build = next(s["with"] for s in base_wf["jobs"]["base"]["steps"] if s.get("id") == "browser")
assert "@${{ steps.base.outputs.digest }}" in browser_build["build-args"]
# Remote ancestry and anonymous existence are checked before downstream builds.
assert 'manifests["base-browser"][:len(base)] != base' in read(".github/workflows/engine-image.yml")
@pytest.mark.parametrize("name", ["engine-image", "task-base-image"])
def test_triggers_caches_and_digest_outputs(name):
wf = workflow(name)
assert "workflow_dispatch" in wf["on"]
assert "src/**" not in wf["on"]["push"]["paths"]
if name == "engine-image":
assert wf["on"]["push"]["tags"] == ["v*"]
assert "plane/task-base.digest" in wf["on"]["push"]["paths"]
else:
assert set(wf["on"]["push"]["paths"]) == {"plane/Dockerfile.base", "pyproject.toml", "uv.lock"}
builds = [s["with"] for job in wf["jobs"].values() for s in job["steps"]
if s.get("uses") == "docker/build-push-action@v6"]
assert builds
for build in builds:
assert "type=gha" in build["cache-from"] and "type=gha" in build["cache-to"]
assert build["push"] == "true"
assert "outputs.digest" in read(f".github/workflows/{name}.yml")
def test_pin_bootstrap_fails_closed():
pins = json.loads(read("plane/task-base.digest"))
assert set(pins) >= {"base", "base-browser"}
for key in ("base", "base-browser"):
if pins[key] is None:
assert "UNPUBLISHED" in pins["note"]
else:
assert re.fullmatch(r"ghcr.io/norandom/ousast-task-base:base-[a-z0-9-]+@sha256:[0-9a-f]{64}", pins[key])
wf = read(".github/workflows/engine-image.yml")
assert "raise SystemExit" in wf and '"inspect", "--raw", ref' in wf
assert "needs: pins" in wf
@pytest.mark.parametrize("case", ["unpublished", "missing", "wrong-ancestry", "valid"])
def test_ci_pin_gate_offline(case, tmp_path, monkeypatch):
"""Exercise the actual workflow Python, with only registry reads replaced."""
import subprocess
pin_step = next(s for s in workflow("engine-image")["jobs"]["pins"]["steps"] if s.get("id") == "pins")
script = pin_step["run"].split("<<'PYTHON'\n", 1)[1].rsplit("PYTHON", 1)[0]
base = "ghcr.io/norandom/ousast-task-base:base-1234567@sha256:" + "a" * 64
browser = "ghcr.io/norandom/ousast-task-base:base-1234567-browser@sha256:" + "b" * 64
(tmp_path / "plane").mkdir()
(tmp_path / "plane/task-base.digest").write_text(json.dumps({
"base": None if case == "unpublished" else base, "base-browser": browser,
}))
output = tmp_path / "output"
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("GITHUB_OUTPUT", str(output))
calls = []
def inspect(argv):
calls.append(argv)
assert argv[:5] == ["docker", "buildx", "imagetools", "inspect", "--raw"]
if case == "missing":
raise subprocess.CalledProcessError(1, argv)
layers = [{"digest": "sha256:shared"}]
if argv[-1] == browser:
if case == "wrong-ancestry":
layers = [{"digest": "sha256:different"}]
layers.append({"digest": "sha256:chromium"})
return json.dumps({"layers": layers}).encode()
monkeypatch.setattr(subprocess, "check_output", inspect)
if case == "valid":
exec(compile(script, "workflow-pin-gate", "exec"), {})
assert "base_digest=sha256:" in output.read_text()
assert "browser_digest=sha256:" in output.read_text()
assert len(calls) == 2
else:
expected = subprocess.CalledProcessError if case == "missing" else SystemExit
with pytest.raises(expected):
exec(compile(script, "workflow-pin-gate", "exec"), {})
if case == "unpublished":
assert not calls
def test_base_preserves_executor_build_toolchains():
common = read("plane/Dockerfile.base").split("FROM ${TASK_BASE_REF} AS base-browser")[0]
commands = " ".join(instructions("plane/Dockerfile.base")).split("FROM ${TASK_BASE_REF} AS base-browser")[0]
install = re.search(r"apt-get install -y --no-install-recommends\s+(.*?)\s+&&", commands).group(1)
packages = set(install.split())
assert packages == {
"nodejs", "npm", "node-typescript", "php-cli", "composer", "maven",
"sqlite3", "bubblewrap", "util-linux", "git", "ca-certificates",
"libstdc++6", "zlib1g",
}
assert "gradle" not in packages
assert "FROM eclipse-temurin:21-jdk-jammy AS jdk" in common
assert "COPY --from=jdk /opt/java/openjdk /opt/java/openjdk" in common
for command in ("javac -version", "npm --version", "composer --version", "mvn --version",
"python3 -m venv /venv", "/venv/bin/pip install"):
assert command in common
assert "joern" not in common.lower()
assert "apt-get clean" in common
for directory in ("doc", "man", "info"):
assert f"path-exclude=/usr/share/{directory}/*" in common
assert f" /usr/share/{directory}/*" in common