mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
images: shared task base, code-last layers, no per-push rebuilds [skip ci]
One base (Temurin 21 JDK, node/npm/tsc, php/composer, maven, sqlite, bwrap) built only when its Dockerfile or dependencies change, plus a browser variant; engine/search/VM images FROM base@sha256 with the code in the last layer; the image workflow runs on dispatch, tags, Dockerfile or pin changes, with a build cache. Executor NPROC bound relative to the current process count. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
7fb061fb69
commit
1b9d1ca85c
@@ -3,18 +3,11 @@ on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ['v*']
|
||||
paths:
|
||||
- 'src/**'
|
||||
- 'benchmarks/learn/engine_trace*.py'
|
||||
- 'benchmarks/learn/engine_task_entry.py'
|
||||
- 'benchmarks/unseen/replay_entry.py'
|
||||
- 'benchmarks/learn/engine_worker_service.py'
|
||||
- 'plane/Dockerfile.*'
|
||||
- 'plane/task-base.digest'
|
||||
- 'Dockerfile'
|
||||
- 'plane/Dockerfile.engine-task'
|
||||
- 'plane/Dockerfile.search-task'
|
||||
- 'ops/frontend-retention/install.py'
|
||||
- 'pyproject.toml'
|
||||
- '.github/workflows/engine-image.yml'
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
@@ -22,7 +15,40 @@ concurrency:
|
||||
group: engine-image-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
pins:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
base_tag: ${{ steps.pins.outputs.base_tag }}
|
||||
base_digest: ${{ steps.pins.outputs.base_digest }}
|
||||
browser_tag: ${{ steps.pins.outputs.browser_tag }}
|
||||
browser_digest: ${{ steps.pins.outputs.browser_digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
# No login: every pinned base must be anonymously pullable.
|
||||
- id: pins
|
||||
run: |
|
||||
python3 - <<'PYTHON'
|
||||
import json, os, re, subprocess
|
||||
from pathlib import Path
|
||||
pins = json.loads(Path("plane/task-base.digest").read_text())
|
||||
manifests = {}
|
||||
for key, output in (("base", "base"), ("base-browser", "browser")):
|
||||
ref = pins.get(key)
|
||||
pattern = r"ghcr.io/norandom/ousast-task-base:(base-[a-z0-9-]+)@(sha256:[0-9a-f]{64})"
|
||||
match = re.fullmatch(pattern, ref or "")
|
||||
if not match:
|
||||
raise SystemExit(f"{key}: unpublished/invalid pin; run task-base-image and update plane/task-base.digest")
|
||||
raw = subprocess.check_output(["docker", "buildx", "imagetools", "inspect", "--raw", ref])
|
||||
manifests[key] = json.loads(raw)["layers"]
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as out:
|
||||
out.write(f"{output}_tag={match[1]}\n{output}_digest={match[2]}\n")
|
||||
base = manifests["base"]
|
||||
if not base or manifests["base-browser"][:len(base)] != base:
|
||||
raise SystemExit("Browser pin does not share the pinned base's layers")
|
||||
PYTHON
|
||||
image:
|
||||
needs: pins
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -30,12 +56,6 @@ jobs:
|
||||
include:
|
||||
- package: openultrasast
|
||||
dockerfile: Dockerfile
|
||||
- package: ousast-engine-task
|
||||
dockerfile: plane/Dockerfile.engine-task
|
||||
target: engine
|
||||
- package: ousast-replay-task
|
||||
dockerfile: plane/Dockerfile.engine-task
|
||||
target: replay
|
||||
- package: ousast-search-task
|
||||
dockerfile: plane/Dockerfile.search-task
|
||||
steps:
|
||||
@@ -58,9 +78,18 @@ jobs:
|
||||
with:
|
||||
context: .
|
||||
file: ${{ matrix.dockerfile }}
|
||||
target: ${{ matrix.target }}
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
build-args: |
|
||||
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
|
||||
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
|
||||
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
|
||||
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=${{ matrix.package }}
|
||||
type=gha,scope=ousast-engine-task
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.package }}
|
||||
tags: |
|
||||
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
||||
${{ steps.names.outputs.image }}:main
|
||||
@@ -70,3 +99,101 @@ jobs:
|
||||
run: |
|
||||
echo "$IMAGE@$DIGEST"
|
||||
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
||||
docker logout ghcr.io
|
||||
docker buildx imagetools inspect "$IMAGE@$DIGEST"
|
||||
engine:
|
||||
needs: pins
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
image: ${{ steps.names.outputs.image }}
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- id: names
|
||||
env:
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
PACKAGE: ousast-engine-task
|
||||
run: |
|
||||
echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||
- id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: plane/Dockerfile.engine-task
|
||||
target: engine
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
build-args: |
|
||||
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
|
||||
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
|
||||
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
|
||||
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
|
||||
cache-from: type=gha,scope=ousast-engine-task
|
||||
cache-to: type=gha,mode=max,scope=ousast-engine-task
|
||||
tags: |
|
||||
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
||||
${{ steps.names.outputs.image }}:main
|
||||
- env:
|
||||
IMAGE: ${{ steps.names.outputs.image }}
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
run: |
|
||||
echo "$IMAGE@$DIGEST"
|
||||
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
||||
docker logout ghcr.io
|
||||
docker buildx imagetools inspect "$IMAGE@$DIGEST"
|
||||
replay:
|
||||
needs: [pins, engine]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- id: names
|
||||
env:
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
PACKAGE: ousast-replay-task
|
||||
run: |
|
||||
echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||
- id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: plane/Dockerfile.engine-task
|
||||
target: replay
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
build-args: |
|
||||
ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }}
|
||||
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
|
||||
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
|
||||
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
|
||||
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=ousast-replay-task
|
||||
type=gha,scope=ousast-engine-task
|
||||
cache-to: type=gha,mode=max,scope=ousast-replay-task
|
||||
tags: |
|
||||
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
||||
${{ steps.names.outputs.image }}:main
|
||||
- env:
|
||||
IMAGE: ${{ steps.names.outputs.image }}
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
run: |
|
||||
echo "$IMAGE@$DIGEST"
|
||||
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
||||
docker logout ghcr.io
|
||||
docker buildx imagetools inspect "$IMAGE@$DIGEST"
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
name: Task base image
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'plane/Dockerfile.base'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
concurrency:
|
||||
group: task-base-image
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
base:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- id: names
|
||||
run: |
|
||||
echo "tag=base-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||
content=$(cat plane/Dockerfile.base pyproject.toml uv.lock | sha256sum | cut -d' ' -f1)
|
||||
echo "content=base-content-$content" >> "$GITHUB_OUTPUT"
|
||||
- id: base
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: plane/Dockerfile.base
|
||||
target: base
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
sbom: false
|
||||
tags: |
|
||||
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}
|
||||
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }}
|
||||
cache-from: type=gha,scope=task-base
|
||||
cache-to: type=gha,mode=max,scope=task-base
|
||||
- id: browser
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: plane/Dockerfile.base
|
||||
target: base-browser
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
provenance: false
|
||||
sbom: false
|
||||
build-args: |
|
||||
TASK_BASE_REF=ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }}
|
||||
tags: |
|
||||
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser
|
||||
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }}-browser
|
||||
cache-from: type=gha,scope=task-base-browser
|
||||
cache-to: type=gha,mode=max,scope=task-base-browser
|
||||
- name: Print manual pin update and verify public access
|
||||
env:
|
||||
BASE: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }}
|
||||
BROWSER: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser@${{ steps.browser.outputs.digest }}
|
||||
run: |
|
||||
python3 - <<'PYTHON' >> "$GITHUB_STEP_SUMMARY"
|
||||
import json, os
|
||||
print("Replace plane/task-base.digest with this JSON after both public checks pass:")
|
||||
print("```json")
|
||||
print(json.dumps({"base": os.environ["BASE"], "base-browser": os.environ["BROWSER"]}, indent=2))
|
||||
print("```")
|
||||
PYTHON
|
||||
docker logout ghcr.io
|
||||
docker buildx imagetools inspect "$BASE"
|
||||
docker buildx imagetools inspect "$BROWSER"
|
||||
+19
-42
@@ -1,26 +1,14 @@
|
||||
# OpenUltraSAST with its CPG engine, in one image.
|
||||
#
|
||||
# The model layer's arbiter is Joern, and `openultrasast.cpg.backend` reaches it by subprocess. Shipping both
|
||||
# in one image is what makes that simple: Joern is on PATH inside, so there is no container-to-container call,
|
||||
# no docker socket, no path translation between host and container, and no second code path in the backend to
|
||||
# get wrong. The alternative -- tool and engine as sibling services -- needs the tool to exec into the other
|
||||
# container, which means mounting the docker socket, and that is a large amount of privilege for a linter.
|
||||
#
|
||||
# The image is big (a JRE plus ~2 GB of Joern). That is the honest cost of bundling an engine, and it buys a
|
||||
# contributor a working tool from `docker compose run` with nothing installed on their machine.
|
||||
|
||||
FROM eclipse-temurin:21-jre-noble
|
||||
LABEL org.opencontainers.image.licenses="Apache-2.0" \
|
||||
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
|
||||
|
||||
# syntax=docker/dockerfile:1
|
||||
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
|
||||
ARG TASK_BASE_TAG=base-unpublished
|
||||
ARG TASK_BASE_DIGEST
|
||||
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS joern-build
|
||||
ENV PATH="/opt/joern-cli:${PATH}"
|
||||
ARG JOERN_VERSION=v4.0.625
|
||||
ARG ARCHIVE=joern-cli-linux-x86_64.zip
|
||||
|
||||
# php-cli is required by php2cpg, which drives PHP-Parser and shells out to a real interpreter. Without it a
|
||||
# PHP tree fails with an opaque "Process exited with code 1" -- measured on this project, not assumed.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
python3 python3-venv python3-pip unzip curl ca-certificates git php-cli \
|
||||
&& apt-get install -y --no-install-recommends curl unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN set -eux; \
|
||||
@@ -32,39 +20,28 @@ RUN set -eux; \
|
||||
unzip -q "${ARCHIVE}" -d /opt; \
|
||||
rm -f "${ARCHIVE}" "${ARCHIVE}.sha512" checksum.sha512; \
|
||||
/opt/joern-cli/joern-parse --help > /dev/null
|
||||
ENV PATH="/opt/joern-cli:${PATH}"
|
||||
|
||||
# This release ships the JS launcher with a .sh suffix; the backend's direct fallback uses
|
||||
# the same extensionless command convention as the other frontends.
|
||||
RUN if [ ! -e /opt/joern-cli/jssrc2cpg ] && [ -x /opt/joern-cli/jssrc2cpg.sh ]; then \
|
||||
ln -s jssrc2cpg.sh /opt/joern-cli/jssrc2cpg; \
|
||||
fi
|
||||
|
||||
COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py
|
||||
RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \
|
||||
&& rm /tmp/frontend-retention-install.py
|
||||
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md ./
|
||||
COPY src ./src
|
||||
COPY benchmarks ./benchmarks
|
||||
|
||||
# The core install is zero-dependency by design; `semantic` adds the tree-sitter grammars the candidate
|
||||
# enumerator uses. The LLM endpoint stays optional and is configured by environment, never baked in.
|
||||
RUN python3 -m venv /venv \
|
||||
&& /venv/bin/pip install --no-cache-dir -e ".[semantic,s3]"
|
||||
ENV PATH="/venv/bin:${PATH}"
|
||||
|
||||
# Analysis never needs root.
|
||||
# Noble may already provide the ubuntu account at UID 1000. Reuse that identity rather than
|
||||
# failing installation or creating a second account with the same UID.
|
||||
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS engine
|
||||
ENV PATH="/opt/joern-cli:${PATH}"
|
||||
COPY --from=joern-build /opt/joern-cli /opt/joern-cli
|
||||
# Create runtime identity before the application layers.
|
||||
RUN if getent passwd 1000 > /dev/null; then \
|
||||
usermod --login ousast --home /home/ousast --move-home "$(getent passwd 1000 | cut -d: -f1)"; \
|
||||
else \
|
||||
useradd --create-home --uid 1000 ousast; \
|
||||
fi \
|
||||
&& chown -R ousast /app
|
||||
else useradd --create-home --uid 1000 ousast; fi
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md LICENSE ./
|
||||
COPY src ./src
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
/venv/bin/pip install --no-deps --no-cache-dir .
|
||||
COPY --chown=1000 benchmarks ./benchmarks
|
||||
RUN chown -R ousast /app
|
||||
USER ousast
|
||||
|
||||
ENTRYPOINT ["python", "-m", "openultrasast.cli"]
|
||||
CMD ["--help"]
|
||||
|
||||
@@ -260,3 +260,102 @@ filesystem quota; programs that hardcode `/tmp` still consume the worker's RAM.
|
||||
Repository clones must use HTTPS `git clone https://github.com/...git` or codeload
|
||||
archives, never `api.github.com`: tasks share one public IP. Search task inputs are
|
||||
presigned checkout archives, so these entrypoints do not clone repositories themselves.
|
||||
|
||||
## Image layers, publication and node cache budget
|
||||
|
||||
The task family has one shared runtime base, `ousast-task-base`, built from the
|
||||
public AX runner v0.3.1. `base-browser` adds Chromium to that exact published base
|
||||
(by digest); engine and replay do not carry Chromium. The VM/kind `openultrasast`
|
||||
image uses the same base and Joern recipe. Replay inherits the published engine
|
||||
digest in CI, so even a cold build cannot accidentally duplicate its Joern layers.
|
||||
|
||||
Java is **Temurin 21 JDK**, copied from `eclipse-temurin:21-jdk-jammy`.
|
||||
`ops/frontend-retention/install.py` explicitly requires Java 21 and invokes
|
||||
`java -cp ... dotty.tools.dotc.Main` using Joern's bundled Scala compiler.
|
||||
The shared JDK also provides `javac` for Maven/Gradle project builds.
|
||||
Curl/unzip and retention build scratch stay in a discarded Joern build stage.
|
||||
Joern stays above the base in the engine image (also used by replay and the VM
|
||||
lane); its compiler jars and retention provenance must not be blindly pruned.
|
||||
Runtime dependencies from `pyproject.toml` (core plus semantic/s3 for the VM lane)
|
||||
precede project code. Package installation uses `--no-deps --no-cache-dir`;
|
||||
its isolated packaging backend is temporary.
|
||||
|
||||
The base retains upstream Python 3.12 with venv/pip. Its explicit apt package list
|
||||
is `nodejs npm node-typescript php-cli composer maven sqlite3 bubblewrap util-linux
|
||||
git ca-certificates libstdc++6 zlib1g`. Composer is the trixie apt package.
|
||||
Chromium is installed only in `base-browser`. Apt caches/lists and docs/man/info
|
||||
are cleaned; dpkg excludes new docs/man/info. Files already in inherited AX
|
||||
layers still occupy space even if deleted.
|
||||
|
||||
Executor preparation supports pip, npm, Composer, Maven and Gradle builds.
|
||||
There is **no installed Gradle binary**: the executor invokes the selected project's
|
||||
`gradlew` through `/bin/sh`, including for wrappers without executable permissions.
|
||||
Gradle-wrapper distribution downloads and dependency downloads go through
|
||||
**public HTTPS from the executor**, subject to its egress policy and deadline.
|
||||
The verifier receives prepared products and performs no dependency downloads.
|
||||
|
||||
Rebuild policy:
|
||||
|
||||
- Dependency/toolchain changes: run **Task base image** (also triggered by changes
|
||||
to `plane/Dockerfile.base`, `pyproject.toml`, or `uv.lock`). It publishes
|
||||
`base-<short sha>` and `base-content-<input hash>` tags, plus their `-browser`
|
||||
variants, and prints immutable digest references. Content tags identify build
|
||||
inputs; they do not claim reproducible upstream apt/PyPI resolution. The lockfile
|
||||
triggers refreshes but pip installs the declared runtime constraints, not a uv
|
||||
frozen environment. Deliberate upstream refreshes use manual dispatch.
|
||||
- After both targets publish, ensure `ousast-task-base` is **public** in GHCR's
|
||||
package settings. Copy the workflow summary's JSON into `plane/task-base.digest`
|
||||
and submit that pin update for normal review. CI does not commit on your behalf.
|
||||
The initial null pins mean **unpublished**, not a usable bootstrap image.
|
||||
- A pin update or `plane/Dockerfile.*`/root Dockerfile change on main runs **Engine
|
||||
image**. For source, entrypoint or retention-installer changes, dispatch it once
|
||||
for the desired revision, or push a `v*` release tag. Source-only merges no longer
|
||||
publish images automatically. Never rebuild per task/run. All builds use scoped
|
||||
GHA caches; pin checks precede builds and require anonymous access plus identical
|
||||
base-layer ancestry for the browser variant.
|
||||
- Final package names must also be public in GHCR. Publication checks anonymous
|
||||
access after logout and fails if the package is private. Set visibility on first
|
||||
publication and rerun as needed. Deploy only the reported `image@sha256:...`
|
||||
references in the cluster profile/Task manifests, never `main` or a release tag.
|
||||
Publication does not automatically change deployed Task pins.
|
||||
|
||||
For a local task build, pass `TASK_BASE_TAG` and `TASK_BASE_DIGEST` from the base
|
||||
pin; search instead takes `TASK_BROWSER_TAG` and `TASK_BROWSER_DIGEST`. Replay's
|
||||
`ENGINE_REF` defaults to its local engine stage; set it to the published engine
|
||||
reference to reuse exactly those layers. When building `base-browser` locally,
|
||||
pass `TASK_BASE_REF=<full base tag@digest>` for the same ancestry as CI.
|
||||
|
||||
The operator's capacity rule is **all images in rotation, unpacked, together
|
||||
at most approximately 9 GiB**, including rollback revisions. Working builds take
|
||||
priority over minimizing an individual executor image; its previous 2.13 GB
|
||||
unpacked (~1.98 GiB) was acceptable. Retire unused Task templates and image
|
||||
revisions before rotating a large base/Joern generation. Shared layers occupy
|
||||
storage once where the runtime deduplicates them, but do not assume that turns
|
||||
a sum above 9 GiB into an acceptable rotation. Record both the sum of unpacked
|
||||
image sizes and actual unique-layer storage. Never prune images used by live tasks.
|
||||
|
||||
Expected footprint (planning estimates only; no images built for this change):
|
||||
|
||||
| Image | Expected unpacked size | Basis / uncertainty |
|
||||
| --- | --- | --- |
|
||||
| Shared base | about 1.3–1.9 GiB | ~454 MB inherited AX layers + ~281 MB JDK baseline + npm/Composer/Maven, language runtimes and venv/extras; apt dependency closure unmeasured |
|
||||
| Browser base | about 1.65–2.45 GiB | Shared base + roughly 350–550 MB Chromium/dependencies |
|
||||
| Search executor/verifier | about 1.7–2.5 GiB | Browser base + application; previous 2.13 GB unpacked is a baseline, not a ceiling |
|
||||
| Engine | roughly 3.5–6 GiB | Restored shared base + Joern; the supplied 2.16 GB compressed engine cannot establish its unpacked size |
|
||||
| Replay | engine + a few KiB | Same engine digest plus replay entrypoint |
|
||||
| VM/kind openultrasast | roughly 3.5–6 GiB plus benchmark assets | Same base/Joern, runtime extras shared, benchmark payload retained |
|
||||
|
||||
An engine + search rotation therefore sums to about **5.2–8.5 GiB** before
|
||||
rollback images. Adding replay as another unpacked image brings the sum to about
|
||||
**8.7–14.5 GiB**; this rotation is not demonstrated to fit 9 GiB. Engine/replay
|
||||
share almost all layers, but the operator must confirm the accounting and measured
|
||||
footprint before selecting a rotation. Extra revisions and the VM image add to
|
||||
the budget if retained there.
|
||||
|
||||
The previous search history included a 1.37 GB apt layer and 281 MB JDK; no
|
||||
per-package apt breakdown is available. Removing system Gradle and docs/caches
|
||||
may save space, while retained build tools and shared Python extras cost space.
|
||||
These ranges are not measured savings or a capacity guarantee. After publication,
|
||||
measure unpacked images and unique layers for the entire intended rotation, and
|
||||
verify build recipes, PHP/JavaScript retention, runner startup and Chromium before
|
||||
rollout. There is no separate <1.5 GiB executor gate.
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# base-browser is built against the published base digest in CI; local builds
|
||||
# default to the base stage. Never independently rebuild its shared ancestry.
|
||||
ARG TASK_BASE_REF=base
|
||||
FROM eclipse-temurin:21-jdk-jammy AS jdk
|
||||
|
||||
FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS base
|
||||
LABEL org.opencontainers.image.licenses="Apache-2.0" \
|
||||
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
|
||||
USER root
|
||||
COPY --from=jdk /opt/java/openjdk /opt/java/openjdk
|
||||
ENV JAVA_HOME=/opt/java/openjdk
|
||||
ENV PATH="/venv/bin:/opt/java/openjdk/bin:${PATH}"
|
||||
# Use upstream /usr/local Python 3.12 (also required by frontend retention).
|
||||
# Its venv/pip and the toolchains below support every executor build recipe.
|
||||
# Gradle projects use their wrapper; no system Gradle is installed.
|
||||
RUN printf 'path-exclude=/usr/share/doc/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/info/*\n' \
|
||||
> /etc/dpkg/dpkg.cfg.d/01-no-docs \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
nodejs npm node-typescript php-cli composer maven sqlite3 bubblewrap util-linux git ca-certificates libstdc++6 zlib1g \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /usr/share/doc/* /usr/share/man/* /usr/share/info/* \
|
||||
/opt/java/openjdk/lib/src.zip /opt/java/openjdk/man /root/.cache
|
||||
# Metadata only: install runtime dependencies, including the VM lane's extras.
|
||||
# No project package or packaging backend enters this shared environment.
|
||||
COPY pyproject.toml /tmp/dependencies/pyproject.toml
|
||||
RUN python3 -c 'import pathlib,tomllib; p=tomllib.loads(pathlib.Path("/tmp/dependencies/pyproject.toml").read_text())["project"]; print("\n".join(p["dependencies"] + p["optional-dependencies"]["semantic"] + p["optional-dependencies"]["s3"]))' > /tmp/requirements.txt \
|
||||
&& python3 -m venv /venv \
|
||||
&& /venv/bin/pip install --no-cache-dir -r /tmp/requirements.txt \
|
||||
&& rm -rf /tmp/dependencies /tmp/requirements.txt /root/.cache \
|
||||
&& python3 --version && java -version && javac -version && node --version && npm --version && tsc --version \
|
||||
&& php --version && composer --version && mvn --version && sqlite3 --version && bwrap --version \
|
||||
&& setpriv --version && prlimit --version && nsenter --version
|
||||
|
||||
FROM ${TASK_BASE_REF} AS base-browser
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends chromium \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /usr/share/doc/* /usr/share/man/* /usr/share/info/* /root/.cache \
|
||||
&& chromium --version
|
||||
@@ -1,23 +1,15 @@
|
||||
# Build from the repository root. AX v0.3.1 starts spec.command immediately;
|
||||
# preserve its runner as PID 1 and put the engine command in the Task manifest.
|
||||
FROM eclipse-temurin:21-jre-jammy AS jre
|
||||
|
||||
FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS engine
|
||||
LABEL org.opencontainers.image.licenses="Apache-2.0" \
|
||||
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
|
||||
|
||||
USER root
|
||||
# syntax=docker/dockerfile:1
|
||||
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
|
||||
ARG TASK_BASE_TAG=base-unpublished
|
||||
ARG TASK_BASE_DIGEST
|
||||
ARG ENGINE_REF=engine
|
||||
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS joern-build
|
||||
ENV PATH="/opt/joern-cli:${PATH}"
|
||||
ARG JOERN_VERSION=v4.0.625
|
||||
ARG ARCHIVE=joern-cli-linux-x86_64.zip
|
||||
|
||||
# The AX runner is Python 3.12 / Debian slim. Copy the Java 21 runtime from
|
||||
# Jammy (older glibc than Debian bookworm) instead of relying on distro JRE versions.
|
||||
COPY --from=jre /opt/java/openjdk /opt/java/openjdk
|
||||
ENV JAVA_HOME=/opt/java/openjdk
|
||||
ENV PATH="/venv/bin:/opt/java/openjdk/bin:/opt/joern-cli:${PATH}"
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
php-cli unzip curl ca-certificates libstdc++6 zlib1g \
|
||||
&& apt-get install -y --no-install-recommends curl unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN set -eux; \
|
||||
@@ -37,24 +29,20 @@ COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py
|
||||
RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \
|
||||
&& rm /tmp/frontend-retention-install.py
|
||||
|
||||
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS engine
|
||||
ENV PATH="/opt/joern-cli:${PATH}"
|
||||
COPY --from=joern-build /opt/joern-cli /opt/joern-cli
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md LICENSE ./
|
||||
COPY src ./src
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
/venv/bin/pip install --no-deps --no-cache-dir .
|
||||
COPY benchmarks/learn/engine_task_entry.py \
|
||||
benchmarks/learn/engine_trace_worker.py \
|
||||
benchmarks/learn/engine_trace_parse.py ./benchmarks/learn/
|
||||
# Worker imports are standard-library plus engine modules. Isolate them from
|
||||
# the AX runner's Python environment and omit PyYAML, boto3 and all extras.
|
||||
RUN python3 -m venv /venv \
|
||||
&& /venv/bin/pip install --no-cache-dir --no-deps . \
|
||||
&& /venv/bin/python -I -c 'import pathlib, runpy, sys; sys.modules.update(yaml=None, boto3=None); sys.path.insert(0, "/app/benchmarks/learn"); p = pathlib.Path("/app/benchmarks/learn/engine_trace_worker.py"); print("Worker import smoke bytes:", len(p.read_bytes())); runpy.run_path(str(p), run_name="engine_worker_import_smoke")'
|
||||
RUN /venv/bin/python -I -c 'import pathlib, runpy, sys; sys.modules.update(yaml=None, boto3=None); sys.path.insert(0, "/app/benchmarks/learn"); p = pathlib.Path("/app/benchmarks/learn/engine_trace_worker.py"); print("Worker import smoke bytes:", len(p.read_bytes())); runpy.run_path(str(p), run_name="engine_worker_import_smoke")'
|
||||
# ENTRYPOINT and CMD belong to the upstream AX runner.
|
||||
|
||||
# Do not set ENTRYPOINT or CMD: both belong to the upstream AX runner.
|
||||
|
||||
FROM engine AS replay
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
FROM ${ENGINE_REF} AS replay
|
||||
COPY benchmarks/unseen/replay_entry.py ./benchmarks/unseen/replay_entry.py
|
||||
RUN /venv/bin/pip install --no-cache-dir . \
|
||||
&& /venv/bin/ousast pre-push --help > /dev/null
|
||||
RUN /venv/bin/ousast pre-push --help > /dev/null
|
||||
|
||||
@@ -1,35 +1,19 @@
|
||||
# Build from the repository root; retain the AX runner entrypoint and command.
|
||||
FROM eclipse-temurin:17-jdk-jammy AS jdk
|
||||
FROM ghcr.io/norandom/ax-task-runner:v0.3.1
|
||||
LABEL org.opencontainers.image.licenses="Apache-2.0" \
|
||||
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
|
||||
# syntax=docker/dockerfile:1
|
||||
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
|
||||
ARG TASK_BROWSER_TAG=base-unpublished-browser
|
||||
ARG TASK_BROWSER_DIGEST
|
||||
FROM ghcr.io/norandom/ousast-task-base:${TASK_BROWSER_TAG}@${TASK_BROWSER_DIGEST}
|
||||
USER root
|
||||
COPY --from=jdk /opt/java/openjdk /opt/java/openjdk
|
||||
ENV JAVA_HOME=/opt/java/openjdk
|
||||
ENV PATH="/venv/bin:/opt/java/openjdk/bin:${PATH}"
|
||||
# dpkg exclusions prevent docs/manpages entering layers, including transitive deps.
|
||||
RUN printf 'path-exclude=/usr/share/doc/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/info/*\n' \
|
||||
> /etc/dpkg/dpkg.cfg.d/01-no-docs \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
python3 python3-venv python3-pip nodejs npm node-typescript php-cli composer maven gradle \
|
||||
sqlite3 bubblewrap util-linux chromium git ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/* /opt/java/openjdk/lib/src.zip /opt/java/openjdk/man
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md LICENSE ./
|
||||
COPY src ./src
|
||||
# Includes the verifier-owned recipes, oracles, and key-free executor entrypoints.
|
||||
RUN python3 -m venv /venv \
|
||||
&& /venv/bin/pip install --no-cache-dir . \
|
||||
&& python3 --version && node --version && tsc --version && php --version \
|
||||
&& javac -version && sqlite3 --version && bwrap --version \
|
||||
&& setpriv --version && prlimit --version && nsenter --version && chromium --version
|
||||
# Dedicated identity for artefacts, apps and Chromium; the verifier keeps root.
|
||||
RUN groupadd --gid 10001 search \
|
||||
&& useradd --create-home --uid 10001 --gid 10001 search
|
||||
# gVisor is the outer boundary; retain inner bwrap when its capability probe succeeds.
|
||||
# The Python base image sets GPG_KEY to the public CPython release-signing fingerprint (not a secret).
|
||||
# Clear it so the executor's guard against inherited credentials stays strict and still passes.
|
||||
# Upstream's public CPython signing fingerprint trips the credential guard.
|
||||
ENV GPG_KEY=
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md LICENSE ./
|
||||
COPY src ./src
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
/venv/bin/pip install --no-deps --no-cache-dir .
|
||||
# Entrypoints and verifier-owned recipes/oracles are included in src.
|
||||
USER root
|
||||
# Do not replace the AX runner's ENTRYPOINT or CMD.
|
||||
# Preserve the AX runner ENTRYPOINT and CMD.
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"base": null,
|
||||
"base-browser": null,
|
||||
"note": "UNPUBLISHED: run task-base-image, then replace with the two full tag@sha256 references from its summary. Never invent digests."
|
||||
}
|
||||
@@ -274,9 +274,28 @@ class InProcessExecutor:
|
||||
}
|
||||
|
||||
def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]:
|
||||
process_limit = None
|
||||
try:
|
||||
uid = os.getuid()
|
||||
current = 0
|
||||
for entry in Path("/proc").iterdir():
|
||||
if not entry.name.isdecimal():
|
||||
continue
|
||||
try:
|
||||
status = (entry / "status").read_text()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
real_uid = int(next(line for line in status.splitlines() if line.startswith("Uid:")).split()[1])
|
||||
current += real_uid == uid
|
||||
process_limit = current + 128
|
||||
except (OSError, ValueError, IndexError, StopIteration):
|
||||
pass
|
||||
|
||||
def constrain() -> None:
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (limits.get("disk_bytes", task_storage.DEFAULT_SCRATCH_BYTES),) * 2)
|
||||
resource.setrlimit(resource.RLIMIT_NPROC, (128, 128))
|
||||
# NPROC counts all processes of the real uid, so an absolute 128 blocks builds on busy hosts.
|
||||
if process_limit is not None:
|
||||
resource.setrlimit(resource.RLIMIT_NPROC, (process_limit,) * 2)
|
||||
resource.setrlimit(resource.RLIMIT_AS, (limits.get("memory_bytes", 256 * 1024**2),) * 2)
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (math.ceil(timeout) + 1,) * 2)
|
||||
|
||||
@@ -391,7 +410,10 @@ class InProcessExecutor:
|
||||
if recipe["recipe"] == "maven":
|
||||
argv += ["package", "-DskipTests"]
|
||||
elif recipe["recipe"] == "gradle":
|
||||
argv += ["assemble"]
|
||||
# The project pins its Gradle version through its wrapper.
|
||||
# Invoke with sh because checkout archives may lose execute bits.
|
||||
wrapper = self._path(str(path.relative_to(self.repo) / "gradlew"))
|
||||
argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"]
|
||||
built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES})
|
||||
if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
|
||||
return {**built, "status": "could_not_build"}
|
||||
|
||||
@@ -4,17 +4,17 @@ import yaml
|
||||
|
||||
|
||||
def test_image_matches_runner_and_publishes_digest():
|
||||
engine = Path("plane/Dockerfile.engine-task").read_text()
|
||||
base = Path("plane/Dockerfile.base").read_text()
|
||||
image = Path("plane/Dockerfile.search-task").read_text()
|
||||
runner = "FROM ghcr.io/norandom/ax-task-runner:v0.3.1"
|
||||
assert runner in engine and runner in image
|
||||
for package in ("python3", "nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"):
|
||||
assert package in image
|
||||
assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1" in base
|
||||
for package in ("nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"):
|
||||
assert package in base
|
||||
assert "python3 -m venv /venv" in base
|
||||
assert image.rsplit("USER ", 1)[1].startswith("root\n")
|
||||
assert "--uid 10001 --gid 10001 search" in image
|
||||
assert "setpriv --version" in image
|
||||
assert "17-jdk" in image and "--no-install-recommends" in image
|
||||
assert "rm -rf /var/lib/apt/lists/" in image
|
||||
assert "setpriv --version" in base
|
||||
assert "21-jdk" in base and "--no-install-recommends" in base
|
||||
assert "rm -rf /var/lib/apt/lists/" in base
|
||||
workflow_text = Path(".github/workflows/engine-image.yml").read_text()
|
||||
entries = yaml.safe_load(workflow_text)["jobs"]["image"]["strategy"]["matrix"]["include"]
|
||||
assert {"package": "ousast-search-task", "dockerfile": "plane/Dockerfile.search-task"} in entries
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -222,3 +224,60 @@ def test_scratch_failure_preserves_exit_and_stderr(workspace, monkeypatch):
|
||||
assert result["exit_code"] is not None
|
||||
assert result["stderr"] == "pip build detail"
|
||||
assert result["reason"].startswith("ScratchLimit:")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("project", [".", "subproject"])
|
||||
def test_preparation_uses_project_gradle_wrapper(workspace, monkeypatch, project):
|
||||
repo = workspace / "checkout"
|
||||
repo.mkdir()
|
||||
root = repo / project
|
||||
root.mkdir(exist_ok=True)
|
||||
# A real local child proves wrapper execution without system Gradle or network.
|
||||
wrapper = root / "gradlew"
|
||||
wrapper.write_text(
|
||||
'test "$1" = "--no-daemon" || exit 1\n'
|
||||
'test "$2" = "--project-dir" || exit 2\n'
|
||||
'test "$4" = "assemble" || exit 3\n'
|
||||
'mkdir -p "$3/build"\n'
|
||||
'printf compiled > "$3/build/product"\n'
|
||||
)
|
||||
wrapper.chmod(0o644)
|
||||
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||
app.prepared_put_url = "https://files.example/output"
|
||||
blobs = []
|
||||
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
|
||||
request = spec()
|
||||
request["demo"]["build"] = {"recipe": "gradle", "arguments": [project]}
|
||||
result = app.submit("prepare_verification", request)
|
||||
assert result["status"] == "ok", result
|
||||
unpacked = workspace / "unpacked"
|
||||
verify_task.extract_checkout(blobs[0], unpacked)
|
||||
assert (unpacked / "checkout" / project / "build/product").read_text() == "compiled"
|
||||
assert json.loads((unpacked / "spec.json").read_text())["demo"]["build"]["recipe"] == "none"
|
||||
|
||||
|
||||
def test_preparation_with_many_host_processes(workspace, monkeypatch):
|
||||
repo = workspace / "checkout"
|
||||
repo.mkdir()
|
||||
(repo / "gradlew").write_text('mkdir -p "$3/build"\nprintf compiled > "$3/build/product"\n')
|
||||
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||
app.prepared_put_url = "https://files.example/output"
|
||||
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: None)
|
||||
request = spec()
|
||||
request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]}
|
||||
children = []
|
||||
try:
|
||||
for _ in range(300):
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
time.sleep(60)
|
||||
os._exit(0)
|
||||
children.append(pid)
|
||||
result = app.submit("prepare_verification", request)
|
||||
assert result["status"] == "ok", result
|
||||
assert (repo / "build/product").read_text() == "compiled"
|
||||
finally:
|
||||
for pid in children:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
for pid in children:
|
||||
os.waitpid(pid, 0)
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Offline image contracts; these do not claim a built image or measured size."""
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def read(name):
|
||||
return (ROOT / name).read_text()
|
||||
|
||||
|
||||
def instructions(name):
|
||||
text = re.sub(r"\\\n\s*", " ", read(name))
|
||||
return [line for line in text.splitlines() if line and not line.startswith("#")]
|
||||
|
||||
|
||||
def workflow(name):
|
||||
# BaseLoader avoids YAML 1.1 interpreting GitHub's `on` as a boolean.
|
||||
return yaml.load(read(f".github/workflows/{name}.yml"), Loader=yaml.BaseLoader)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,variant", [
|
||||
("plane/Dockerfile.engine-task", "BASE"),
|
||||
("plane/Dockerfile.search-task", "BROWSER"),
|
||||
("Dockerfile", "BASE"),
|
||||
])
|
||||
def test_digest_required_and_code_last(name, variant):
|
||||
lines = instructions(name)
|
||||
assert f"ARG TASK_{variant}_DIGEST" in lines
|
||||
ref = f"ghcr.io/norandom/ousast-task-base:${{TASK_{variant}_TAG}}@${{TASK_{variant}_DIGEST}}"
|
||||
assert any(line.startswith(f"FROM {ref}") for line in lines)
|
||||
code = next(i for i, line in enumerate(lines) if line.startswith("COPY src "))
|
||||
for i, line in enumerate(lines):
|
||||
if "apt-get" in line or "--from=joern-build" in line or "frontend-retention-install.py" in line:
|
||||
assert i < code
|
||||
if "pip install" in line:
|
||||
assert "--no-deps --no-cache-dir ." in line
|
||||
assert i > code
|
||||
assert not any("venv /venv" in line for line in lines)
|
||||
|
||||
|
||||
def test_shared_ancestry_and_replay_exact_engine():
|
||||
base = read("plane/Dockerfile.base")
|
||||
assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS base" in base
|
||||
assert "FROM ${TASK_BASE_REF} AS base-browser" in base
|
||||
common, browser = base.split("FROM ${TASK_BASE_REF} AS base-browser")
|
||||
assert "chromium" not in common and "chromium" in browser
|
||||
assert "21-jdk-jammy" in common
|
||||
assert "COPY src" not in base and 'p["dependencies"]' in base
|
||||
assert "pip install --no-cache-dir -r /tmp/requirements.txt" in base
|
||||
engine = read("plane/Dockerfile.engine-task")
|
||||
assert "ARG ENGINE_REF=engine" in engine and "FROM ${ENGINE_REF} AS replay" in engine
|
||||
wf = workflow("engine-image")
|
||||
assert wf["jobs"]["replay"]["needs"] == ["pins", "engine"]
|
||||
builds = [s["with"] for s in wf["jobs"]["replay"]["steps"] if s.get("uses") == "docker/build-push-action@v6"]
|
||||
assert "ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }}" in builds[0]["build-args"]
|
||||
base_wf = workflow("task-base-image")
|
||||
browser_build = next(s["with"] for s in base_wf["jobs"]["base"]["steps"] if s.get("id") == "browser")
|
||||
assert "@${{ steps.base.outputs.digest }}" in browser_build["build-args"]
|
||||
# Remote ancestry and anonymous existence are checked before downstream builds.
|
||||
assert 'manifests["base-browser"][:len(base)] != base' in read(".github/workflows/engine-image.yml")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["engine-image", "task-base-image"])
|
||||
def test_triggers_caches_and_digest_outputs(name):
|
||||
wf = workflow(name)
|
||||
assert "workflow_dispatch" in wf["on"]
|
||||
assert "src/**" not in wf["on"]["push"]["paths"]
|
||||
if name == "engine-image":
|
||||
assert wf["on"]["push"]["tags"] == ["v*"]
|
||||
assert "plane/task-base.digest" in wf["on"]["push"]["paths"]
|
||||
else:
|
||||
assert set(wf["on"]["push"]["paths"]) == {"plane/Dockerfile.base", "pyproject.toml", "uv.lock"}
|
||||
builds = [s["with"] for job in wf["jobs"].values() for s in job["steps"]
|
||||
if s.get("uses") == "docker/build-push-action@v6"]
|
||||
assert builds
|
||||
for build in builds:
|
||||
assert "type=gha" in build["cache-from"] and "type=gha" in build["cache-to"]
|
||||
assert build["push"] == "true"
|
||||
assert "outputs.digest" in read(f".github/workflows/{name}.yml")
|
||||
|
||||
|
||||
def test_pin_bootstrap_fails_closed():
|
||||
pins = json.loads(read("plane/task-base.digest"))
|
||||
assert set(pins) >= {"base", "base-browser"}
|
||||
for key in ("base", "base-browser"):
|
||||
if pins[key] is None:
|
||||
assert "UNPUBLISHED" in pins["note"]
|
||||
else:
|
||||
assert re.fullmatch(r"ghcr.io/norandom/ousast-task-base:base-[a-z0-9-]+@sha256:[0-9a-f]{64}", pins[key])
|
||||
wf = read(".github/workflows/engine-image.yml")
|
||||
assert "raise SystemExit" in wf and '"inspect", "--raw", ref' in wf
|
||||
assert "needs: pins" in wf
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["unpublished", "missing", "wrong-ancestry", "valid"])
|
||||
def test_ci_pin_gate_offline(case, tmp_path, monkeypatch):
|
||||
"""Exercise the actual workflow Python, with only registry reads replaced."""
|
||||
import subprocess
|
||||
|
||||
pin_step = next(s for s in workflow("engine-image")["jobs"]["pins"]["steps"] if s.get("id") == "pins")
|
||||
script = pin_step["run"].split("<<'PYTHON'\n", 1)[1].rsplit("PYTHON", 1)[0]
|
||||
base = "ghcr.io/norandom/ousast-task-base:base-1234567@sha256:" + "a" * 64
|
||||
browser = "ghcr.io/norandom/ousast-task-base:base-1234567-browser@sha256:" + "b" * 64
|
||||
(tmp_path / "plane").mkdir()
|
||||
(tmp_path / "plane/task-base.digest").write_text(json.dumps({
|
||||
"base": None if case == "unpublished" else base, "base-browser": browser,
|
||||
}))
|
||||
output = tmp_path / "output"
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("GITHUB_OUTPUT", str(output))
|
||||
calls = []
|
||||
|
||||
def inspect(argv):
|
||||
calls.append(argv)
|
||||
assert argv[:5] == ["docker", "buildx", "imagetools", "inspect", "--raw"]
|
||||
if case == "missing":
|
||||
raise subprocess.CalledProcessError(1, argv)
|
||||
layers = [{"digest": "sha256:shared"}]
|
||||
if argv[-1] == browser:
|
||||
if case == "wrong-ancestry":
|
||||
layers = [{"digest": "sha256:different"}]
|
||||
layers.append({"digest": "sha256:chromium"})
|
||||
return json.dumps({"layers": layers}).encode()
|
||||
|
||||
monkeypatch.setattr(subprocess, "check_output", inspect)
|
||||
if case == "valid":
|
||||
exec(compile(script, "workflow-pin-gate", "exec"), {})
|
||||
assert "base_digest=sha256:" in output.read_text()
|
||||
assert "browser_digest=sha256:" in output.read_text()
|
||||
assert len(calls) == 2
|
||||
else:
|
||||
expected = subprocess.CalledProcessError if case == "missing" else SystemExit
|
||||
with pytest.raises(expected):
|
||||
exec(compile(script, "workflow-pin-gate", "exec"), {})
|
||||
if case == "unpublished":
|
||||
assert not calls
|
||||
|
||||
|
||||
def test_base_preserves_executor_build_toolchains():
|
||||
common = read("plane/Dockerfile.base").split("FROM ${TASK_BASE_REF} AS base-browser")[0]
|
||||
commands = " ".join(instructions("plane/Dockerfile.base")).split("FROM ${TASK_BASE_REF} AS base-browser")[0]
|
||||
install = re.search(r"apt-get install -y --no-install-recommends\s+(.*?)\s+&&", commands).group(1)
|
||||
packages = set(install.split())
|
||||
assert packages == {
|
||||
"nodejs", "npm", "node-typescript", "php-cli", "composer", "maven",
|
||||
"sqlite3", "bubblewrap", "util-linux", "git", "ca-certificates",
|
||||
"libstdc++6", "zlib1g",
|
||||
}
|
||||
assert "gradle" not in packages
|
||||
assert "FROM eclipse-temurin:21-jdk-jammy AS jdk" in common
|
||||
assert "COPY --from=jdk /opt/java/openjdk /opt/java/openjdk" in common
|
||||
for command in ("javac -version", "npm --version", "composer --version", "mvn --version",
|
||||
"python3 -m venv /venv", "/venv/bin/pip install"):
|
||||
assert command in common
|
||||
assert "joern" not in common.lower()
|
||||
assert "apt-get clean" in common
|
||||
for directory in ("doc", "man", "info"):
|
||||
assert f"path-exclude=/usr/share/{directory}/*" in common
|
||||
assert f" /usr/share/{directory}/*" in common
|
||||
Reference in New Issue
Block a user