mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
merge(image-layering): shared task base and code-last layering [skip ci]
Co-Authored-By: Claude Fable 5.1 <[email protected]> # Conflicts: # tests/test_search_task_storage.py
This commit is contained in:
@@ -3,18 +3,11 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
|
tags: ['v*']
|
||||||
paths:
|
paths:
|
||||||
- 'src/**'
|
- 'plane/Dockerfile.*'
|
||||||
- 'benchmarks/learn/engine_trace*.py'
|
- 'plane/task-base.digest'
|
||||||
- 'benchmarks/learn/engine_task_entry.py'
|
|
||||||
- 'benchmarks/unseen/replay_entry.py'
|
|
||||||
- 'benchmarks/learn/engine_worker_service.py'
|
|
||||||
- 'Dockerfile'
|
- 'Dockerfile'
|
||||||
- 'plane/Dockerfile.engine-task'
|
|
||||||
- 'plane/Dockerfile.search-task'
|
|
||||||
- 'ops/frontend-retention/install.py'
|
|
||||||
- 'pyproject.toml'
|
|
||||||
- '.github/workflows/engine-image.yml'
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
packages: write
|
packages: write
|
||||||
@@ -22,7 +15,40 @@ concurrency:
|
|||||||
group: engine-image-${{ github.ref }}
|
group: engine-image-${{ github.ref }}
|
||||||
cancel-in-progress: false
|
cancel-in-progress: false
|
||||||
jobs:
|
jobs:
|
||||||
|
pins:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
base_tag: ${{ steps.pins.outputs.base_tag }}
|
||||||
|
base_digest: ${{ steps.pins.outputs.base_digest }}
|
||||||
|
browser_tag: ${{ steps.pins.outputs.browser_tag }}
|
||||||
|
browser_digest: ${{ steps.pins.outputs.browser_digest }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v5
|
||||||
|
- uses: docker/setup-buildx-action@v3
|
||||||
|
# No login: every pinned base must be anonymously pullable.
|
||||||
|
- id: pins
|
||||||
|
run: |
|
||||||
|
python3 - <<'PYTHON'
|
||||||
|
import json, os, re, subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
pins = json.loads(Path("plane/task-base.digest").read_text())
|
||||||
|
manifests = {}
|
||||||
|
for key, output in (("base", "base"), ("base-browser", "browser")):
|
||||||
|
ref = pins.get(key)
|
||||||
|
pattern = r"ghcr.io/norandom/ousast-task-base:(base-[a-z0-9-]+)@(sha256:[0-9a-f]{64})"
|
||||||
|
match = re.fullmatch(pattern, ref or "")
|
||||||
|
if not match:
|
||||||
|
raise SystemExit(f"{key}: unpublished/invalid pin; run task-base-image and update plane/task-base.digest")
|
||||||
|
raw = subprocess.check_output(["docker", "buildx", "imagetools", "inspect", "--raw", ref])
|
||||||
|
manifests[key] = json.loads(raw)["layers"]
|
||||||
|
with open(os.environ["GITHUB_OUTPUT"], "a") as out:
|
||||||
|
out.write(f"{output}_tag={match[1]}\n{output}_digest={match[2]}\n")
|
||||||
|
base = manifests["base"]
|
||||||
|
if not base or manifests["base-browser"][:len(base)] != base:
|
||||||
|
raise SystemExit("Browser pin does not share the pinned base's layers")
|
||||||
|
PYTHON
|
||||||
image:
|
image:
|
||||||
|
needs: pins
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
@@ -30,12 +56,6 @@ jobs:
|
|||||||
include:
|
include:
|
||||||
- package: openultrasast
|
- package: openultrasast
|
||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
- package: ousast-engine-task
|
|
||||||
dockerfile: plane/Dockerfile.engine-task
|
|
||||||
target: engine
|
|
||||||
- package: ousast-replay-task
|
|
||||||
dockerfile: plane/Dockerfile.engine-task
|
|
||||||
target: replay
|
|
||||||
- package: ousast-search-task
|
- package: ousast-search-task
|
||||||
dockerfile: plane/Dockerfile.search-task
|
dockerfile: plane/Dockerfile.search-task
|
||||||
steps:
|
steps:
|
||||||
@@ -58,9 +78,18 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
file: ${{ matrix.dockerfile }}
|
file: ${{ matrix.dockerfile }}
|
||||||
target: ${{ matrix.target }}
|
|
||||||
platforms: linux/amd64
|
platforms: linux/amd64
|
||||||
push: true
|
push: true
|
||||||
|
provenance: false
|
||||||
|
build-args: |
|
||||||
|
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
|
||||||
|
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
|
||||||
|
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
|
||||||
|
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
|
||||||
|
cache-from: |
|
||||||
|
type=gha,scope=${{ matrix.package }}
|
||||||
|
type=gha,scope=ousast-engine-task
|
||||||
|
cache-to: type=gha,mode=max,scope=${{ matrix.package }}
|
||||||
tags: |
|
tags: |
|
||||||
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
||||||
${{ steps.names.outputs.image }}:main
|
${{ steps.names.outputs.image }}:main
|
||||||
@@ -70,3 +99,101 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
echo "$IMAGE@$DIGEST"
|
echo "$IMAGE@$DIGEST"
|
||||||
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
docker logout ghcr.io
|
||||||
|
docker buildx imagetools inspect "$IMAGE@$DIGEST"
|
||||||
|
engine:
|
||||||
|
needs: pins
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
image: ${{ steps.names.outputs.image }}
|
||||||
|
digest: ${{ steps.build.outputs.digest }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v5
|
||||||
|
- uses: docker/setup-buildx-action@v3
|
||||||
|
- uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
- id: names
|
||||||
|
env:
|
||||||
|
OWNER: ${{ github.repository_owner }}
|
||||||
|
PACKAGE: ousast-engine-task
|
||||||
|
run: |
|
||||||
|
echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||||
|
- id: build
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: plane/Dockerfile.engine-task
|
||||||
|
target: engine
|
||||||
|
platforms: linux/amd64
|
||||||
|
push: true
|
||||||
|
provenance: false
|
||||||
|
build-args: |
|
||||||
|
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
|
||||||
|
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
|
||||||
|
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
|
||||||
|
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
|
||||||
|
cache-from: type=gha,scope=ousast-engine-task
|
||||||
|
cache-to: type=gha,mode=max,scope=ousast-engine-task
|
||||||
|
tags: |
|
||||||
|
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
||||||
|
${{ steps.names.outputs.image }}:main
|
||||||
|
- env:
|
||||||
|
IMAGE: ${{ steps.names.outputs.image }}
|
||||||
|
DIGEST: ${{ steps.build.outputs.digest }}
|
||||||
|
run: |
|
||||||
|
echo "$IMAGE@$DIGEST"
|
||||||
|
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
docker logout ghcr.io
|
||||||
|
docker buildx imagetools inspect "$IMAGE@$DIGEST"
|
||||||
|
replay:
|
||||||
|
needs: [pins, engine]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v5
|
||||||
|
- uses: docker/setup-buildx-action@v3
|
||||||
|
- uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
- id: names
|
||||||
|
env:
|
||||||
|
OWNER: ${{ github.repository_owner }}
|
||||||
|
PACKAGE: ousast-replay-task
|
||||||
|
run: |
|
||||||
|
echo "image=ghcr.io/${OWNER,,}/$PACKAGE" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "sha=sha-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||||
|
- id: build
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: plane/Dockerfile.engine-task
|
||||||
|
target: replay
|
||||||
|
platforms: linux/amd64
|
||||||
|
push: true
|
||||||
|
provenance: false
|
||||||
|
build-args: |
|
||||||
|
ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }}
|
||||||
|
TASK_BASE_TAG=${{ needs.pins.outputs.base_tag }}
|
||||||
|
TASK_BASE_DIGEST=${{ needs.pins.outputs.base_digest }}
|
||||||
|
TASK_BROWSER_TAG=${{ needs.pins.outputs.browser_tag }}
|
||||||
|
TASK_BROWSER_DIGEST=${{ needs.pins.outputs.browser_digest }}
|
||||||
|
cache-from: |
|
||||||
|
type=gha,scope=ousast-replay-task
|
||||||
|
type=gha,scope=ousast-engine-task
|
||||||
|
cache-to: type=gha,mode=max,scope=ousast-replay-task
|
||||||
|
tags: |
|
||||||
|
${{ steps.names.outputs.image }}:${{ steps.names.outputs.sha }}
|
||||||
|
${{ steps.names.outputs.image }}:main
|
||||||
|
- env:
|
||||||
|
IMAGE: ${{ steps.names.outputs.image }}
|
||||||
|
DIGEST: ${{ steps.build.outputs.digest }}
|
||||||
|
run: |
|
||||||
|
echo "$IMAGE@$DIGEST"
|
||||||
|
echo "$IMAGE@$DIGEST" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
docker logout ghcr.io
|
||||||
|
docker buildx imagetools inspect "$IMAGE@$DIGEST"
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
name: Task base image
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
paths:
|
||||||
|
- 'plane/Dockerfile.base'
|
||||||
|
- 'pyproject.toml'
|
||||||
|
- 'uv.lock'
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
concurrency:
|
||||||
|
group: task-base-image
|
||||||
|
cancel-in-progress: false
|
||||||
|
jobs:
|
||||||
|
base:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v5
|
||||||
|
- uses: docker/setup-buildx-action@v3
|
||||||
|
- uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
- id: names
|
||||||
|
run: |
|
||||||
|
echo "tag=base-${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
|
||||||
|
content=$(cat plane/Dockerfile.base pyproject.toml uv.lock | sha256sum | cut -d' ' -f1)
|
||||||
|
echo "content=base-content-$content" >> "$GITHUB_OUTPUT"
|
||||||
|
- id: base
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: plane/Dockerfile.base
|
||||||
|
target: base
|
||||||
|
platforms: linux/amd64
|
||||||
|
push: true
|
||||||
|
provenance: false
|
||||||
|
sbom: false
|
||||||
|
tags: |
|
||||||
|
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}
|
||||||
|
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }}
|
||||||
|
cache-from: type=gha,scope=task-base
|
||||||
|
cache-to: type=gha,mode=max,scope=task-base
|
||||||
|
- id: browser
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: plane/Dockerfile.base
|
||||||
|
target: base-browser
|
||||||
|
platforms: linux/amd64
|
||||||
|
push: true
|
||||||
|
provenance: false
|
||||||
|
sbom: false
|
||||||
|
build-args: |
|
||||||
|
TASK_BASE_REF=ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }}
|
||||||
|
tags: |
|
||||||
|
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser
|
||||||
|
ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.content }}-browser
|
||||||
|
cache-from: type=gha,scope=task-base-browser
|
||||||
|
cache-to: type=gha,mode=max,scope=task-base-browser
|
||||||
|
- name: Print manual pin update and verify public access
|
||||||
|
env:
|
||||||
|
BASE: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}@${{ steps.base.outputs.digest }}
|
||||||
|
BROWSER: ghcr.io/norandom/ousast-task-base:${{ steps.names.outputs.tag }}-browser@${{ steps.browser.outputs.digest }}
|
||||||
|
run: |
|
||||||
|
python3 - <<'PYTHON' >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
import json, os
|
||||||
|
print("Replace plane/task-base.digest with this JSON after both public checks pass:")
|
||||||
|
print("```json")
|
||||||
|
print(json.dumps({"base": os.environ["BASE"], "base-browser": os.environ["BROWSER"]}, indent=2))
|
||||||
|
print("```")
|
||||||
|
PYTHON
|
||||||
|
docker logout ghcr.io
|
||||||
|
docker buildx imagetools inspect "$BASE"
|
||||||
|
docker buildx imagetools inspect "$BROWSER"
|
||||||
+19
-42
@@ -1,26 +1,14 @@
|
|||||||
# OpenUltraSAST with its CPG engine, in one image.
|
# syntax=docker/dockerfile:1
|
||||||
#
|
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
|
||||||
# The model layer's arbiter is Joern, and `openultrasast.cpg.backend` reaches it by subprocess. Shipping both
|
ARG TASK_BASE_TAG=base-unpublished
|
||||||
# in one image is what makes that simple: Joern is on PATH inside, so there is no container-to-container call,
|
ARG TASK_BASE_DIGEST
|
||||||
# no docker socket, no path translation between host and container, and no second code path in the backend to
|
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS joern-build
|
||||||
# get wrong. The alternative -- tool and engine as sibling services -- needs the tool to exec into the other
|
ENV PATH="/opt/joern-cli:${PATH}"
|
||||||
# container, which means mounting the docker socket, and that is a large amount of privilege for a linter.
|
|
||||||
#
|
|
||||||
# The image is big (a JRE plus ~2 GB of Joern). That is the honest cost of bundling an engine, and it buys a
|
|
||||||
# contributor a working tool from `docker compose run` with nothing installed on their machine.
|
|
||||||
|
|
||||||
FROM eclipse-temurin:21-jre-noble
|
|
||||||
LABEL org.opencontainers.image.licenses="Apache-2.0" \
|
|
||||||
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
|
|
||||||
|
|
||||||
ARG JOERN_VERSION=v4.0.625
|
ARG JOERN_VERSION=v4.0.625
|
||||||
ARG ARCHIVE=joern-cli-linux-x86_64.zip
|
ARG ARCHIVE=joern-cli-linux-x86_64.zip
|
||||||
|
|
||||||
# php-cli is required by php2cpg, which drives PHP-Parser and shells out to a real interpreter. Without it a
|
|
||||||
# PHP tree fails with an opaque "Process exited with code 1" -- measured on this project, not assumed.
|
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends \
|
&& apt-get install -y --no-install-recommends curl unzip \
|
||||||
python3 python3-venv python3-pip unzip curl ca-certificates git php-cli \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN set -eux; \
|
RUN set -eux; \
|
||||||
@@ -32,39 +20,28 @@ RUN set -eux; \
|
|||||||
unzip -q "${ARCHIVE}" -d /opt; \
|
unzip -q "${ARCHIVE}" -d /opt; \
|
||||||
rm -f "${ARCHIVE}" "${ARCHIVE}.sha512" checksum.sha512; \
|
rm -f "${ARCHIVE}" "${ARCHIVE}.sha512" checksum.sha512; \
|
||||||
/opt/joern-cli/joern-parse --help > /dev/null
|
/opt/joern-cli/joern-parse --help > /dev/null
|
||||||
ENV PATH="/opt/joern-cli:${PATH}"
|
|
||||||
|
|
||||||
# This release ships the JS launcher with a .sh suffix; the backend's direct fallback uses
|
|
||||||
# the same extensionless command convention as the other frontends.
|
|
||||||
RUN if [ ! -e /opt/joern-cli/jssrc2cpg ] && [ -x /opt/joern-cli/jssrc2cpg.sh ]; then \
|
RUN if [ ! -e /opt/joern-cli/jssrc2cpg ] && [ -x /opt/joern-cli/jssrc2cpg.sh ]; then \
|
||||||
ln -s jssrc2cpg.sh /opt/joern-cli/jssrc2cpg; \
|
ln -s jssrc2cpg.sh /opt/joern-cli/jssrc2cpg; \
|
||||||
fi
|
fi
|
||||||
|
|
||||||
COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py
|
COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py
|
||||||
RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \
|
RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \
|
||||||
&& rm /tmp/frontend-retention-install.py
|
&& rm /tmp/frontend-retention-install.py
|
||||||
|
|
||||||
WORKDIR /app
|
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS engine
|
||||||
COPY pyproject.toml README.md ./
|
ENV PATH="/opt/joern-cli:${PATH}"
|
||||||
COPY src ./src
|
COPY --from=joern-build /opt/joern-cli /opt/joern-cli
|
||||||
COPY benchmarks ./benchmarks
|
# Create runtime identity before the application layers.
|
||||||
|
|
||||||
# The core install is zero-dependency by design; `semantic` adds the tree-sitter grammars the candidate
|
|
||||||
# enumerator uses. The LLM endpoint stays optional and is configured by environment, never baked in.
|
|
||||||
RUN python3 -m venv /venv \
|
|
||||||
&& /venv/bin/pip install --no-cache-dir -e ".[semantic,s3]"
|
|
||||||
ENV PATH="/venv/bin:${PATH}"
|
|
||||||
|
|
||||||
# Analysis never needs root.
|
|
||||||
# Noble may already provide the ubuntu account at UID 1000. Reuse that identity rather than
|
|
||||||
# failing installation or creating a second account with the same UID.
|
|
||||||
RUN if getent passwd 1000 > /dev/null; then \
|
RUN if getent passwd 1000 > /dev/null; then \
|
||||||
usermod --login ousast --home /home/ousast --move-home "$(getent passwd 1000 | cut -d: -f1)"; \
|
usermod --login ousast --home /home/ousast --move-home "$(getent passwd 1000 | cut -d: -f1)"; \
|
||||||
else \
|
else useradd --create-home --uid 1000 ousast; fi
|
||||||
useradd --create-home --uid 1000 ousast; \
|
WORKDIR /app
|
||||||
fi \
|
COPY pyproject.toml README.md LICENSE ./
|
||||||
&& chown -R ousast /app
|
COPY src ./src
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||||
|
/venv/bin/pip install --no-deps --no-cache-dir .
|
||||||
|
COPY --chown=1000 benchmarks ./benchmarks
|
||||||
|
RUN chown -R ousast /app
|
||||||
USER ousast
|
USER ousast
|
||||||
|
|
||||||
ENTRYPOINT ["python", "-m", "openultrasast.cli"]
|
ENTRYPOINT ["python", "-m", "openultrasast.cli"]
|
||||||
CMD ["--help"]
|
CMD ["--help"]
|
||||||
|
|||||||
@@ -260,3 +260,102 @@ filesystem quota; programs that hardcode `/tmp` still consume the worker's RAM.
|
|||||||
Repository clones must use HTTPS `git clone https://github.com/...git` or codeload
|
Repository clones must use HTTPS `git clone https://github.com/...git` or codeload
|
||||||
archives, never `api.github.com`: tasks share one public IP. Search task inputs are
|
archives, never `api.github.com`: tasks share one public IP. Search task inputs are
|
||||||
presigned checkout archives, so these entrypoints do not clone repositories themselves.
|
presigned checkout archives, so these entrypoints do not clone repositories themselves.
|
||||||
|
|
||||||
|
## Image layers, publication and node cache budget
|
||||||
|
|
||||||
|
The task family has one shared runtime base, `ousast-task-base`, built from the
|
||||||
|
public AX runner v0.3.1. `base-browser` adds Chromium to that exact published base
|
||||||
|
(by digest); engine and replay do not carry Chromium. The VM/kind `openultrasast`
|
||||||
|
image uses the same base and Joern recipe. Replay inherits the published engine
|
||||||
|
digest in CI, so even a cold build cannot accidentally duplicate its Joern layers.
|
||||||
|
|
||||||
|
Java is **Temurin 21 JDK**, copied from `eclipse-temurin:21-jdk-jammy`.
|
||||||
|
`ops/frontend-retention/install.py` explicitly requires Java 21 and invokes
|
||||||
|
`java -cp ... dotty.tools.dotc.Main` using Joern's bundled Scala compiler.
|
||||||
|
The shared JDK also provides `javac` for Maven/Gradle project builds.
|
||||||
|
Curl/unzip and retention build scratch stay in a discarded Joern build stage.
|
||||||
|
Joern stays above the base in the engine image (also used by replay and the VM
|
||||||
|
lane); its compiler jars and retention provenance must not be blindly pruned.
|
||||||
|
Runtime dependencies from `pyproject.toml` (core plus semantic/s3 for the VM lane)
|
||||||
|
precede project code. Package installation uses `--no-deps --no-cache-dir`;
|
||||||
|
its isolated packaging backend is temporary.
|
||||||
|
|
||||||
|
The base retains upstream Python 3.12 with venv/pip. Its explicit apt package list
|
||||||
|
is `nodejs npm node-typescript php-cli composer maven sqlite3 bubblewrap util-linux
|
||||||
|
git ca-certificates libstdc++6 zlib1g`. Composer is the trixie apt package.
|
||||||
|
Chromium is installed only in `base-browser`. Apt caches/lists and docs/man/info
|
||||||
|
are cleaned; dpkg excludes new docs/man/info. Files already in inherited AX
|
||||||
|
layers still occupy space even if deleted.
|
||||||
|
|
||||||
|
Executor preparation supports pip, npm, Composer, Maven and Gradle builds.
|
||||||
|
There is **no installed Gradle binary**: the executor invokes the selected project's
|
||||||
|
`gradlew` through `/bin/sh`, including for wrappers without executable permissions.
|
||||||
|
Gradle-wrapper distribution downloads and dependency downloads go through
|
||||||
|
**public HTTPS from the executor**, subject to its egress policy and deadline.
|
||||||
|
The verifier receives prepared products and performs no dependency downloads.
|
||||||
|
|
||||||
|
Rebuild policy:
|
||||||
|
|
||||||
|
- Dependency/toolchain changes: run **Task base image** (also triggered by changes
|
||||||
|
to `plane/Dockerfile.base`, `pyproject.toml`, or `uv.lock`). It publishes
|
||||||
|
`base-<short sha>` and `base-content-<input hash>` tags, plus their `-browser`
|
||||||
|
variants, and prints immutable digest references. Content tags identify build
|
||||||
|
inputs; they do not claim reproducible upstream apt/PyPI resolution. The lockfile
|
||||||
|
triggers refreshes but pip installs the declared runtime constraints, not a uv
|
||||||
|
frozen environment. Deliberate upstream refreshes use manual dispatch.
|
||||||
|
- After both targets publish, ensure `ousast-task-base` is **public** in GHCR's
|
||||||
|
package settings. Copy the workflow summary's JSON into `plane/task-base.digest`
|
||||||
|
and submit that pin update for normal review. CI does not commit on your behalf.
|
||||||
|
The initial null pins mean **unpublished**, not a usable bootstrap image.
|
||||||
|
- A pin update or `plane/Dockerfile.*`/root Dockerfile change on main runs **Engine
|
||||||
|
image**. For source, entrypoint or retention-installer changes, dispatch it once
|
||||||
|
for the desired revision, or push a `v*` release tag. Source-only merges no longer
|
||||||
|
publish images automatically. Never rebuild per task/run. All builds use scoped
|
||||||
|
GHA caches; pin checks precede builds and require anonymous access plus identical
|
||||||
|
base-layer ancestry for the browser variant.
|
||||||
|
- Final package names must also be public in GHCR. Publication checks anonymous
|
||||||
|
access after logout and fails if the package is private. Set visibility on first
|
||||||
|
publication and rerun as needed. Deploy only the reported `image@sha256:...`
|
||||||
|
references in the cluster profile/Task manifests, never `main` or a release tag.
|
||||||
|
Publication does not automatically change deployed Task pins.
|
||||||
|
|
||||||
|
For a local task build, pass `TASK_BASE_TAG` and `TASK_BASE_DIGEST` from the base
|
||||||
|
pin; search instead takes `TASK_BROWSER_TAG` and `TASK_BROWSER_DIGEST`. Replay's
|
||||||
|
`ENGINE_REF` defaults to its local engine stage; set it to the published engine
|
||||||
|
reference to reuse exactly those layers. When building `base-browser` locally,
|
||||||
|
pass `TASK_BASE_REF=<full base tag@digest>` for the same ancestry as CI.
|
||||||
|
|
||||||
|
The operator's capacity rule is **all images in rotation, unpacked, together
|
||||||
|
at most approximately 9 GiB**, including rollback revisions. Working builds take
|
||||||
|
priority over minimizing an individual executor image; its previous 2.13 GB
|
||||||
|
unpacked (~1.98 GiB) was acceptable. Retire unused Task templates and image
|
||||||
|
revisions before rotating a large base/Joern generation. Shared layers occupy
|
||||||
|
storage once where the runtime deduplicates them, but do not assume that turns
|
||||||
|
a sum above 9 GiB into an acceptable rotation. Record both the sum of unpacked
|
||||||
|
image sizes and actual unique-layer storage. Never prune images used by live tasks.
|
||||||
|
|
||||||
|
Expected footprint (planning estimates only; no images built for this change):
|
||||||
|
|
||||||
|
| Image | Expected unpacked size | Basis / uncertainty |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Shared base | about 1.3–1.9 GiB | ~454 MB inherited AX layers + ~281 MB JDK baseline + npm/Composer/Maven, language runtimes and venv/extras; apt dependency closure unmeasured |
|
||||||
|
| Browser base | about 1.65–2.45 GiB | Shared base + roughly 350–550 MB Chromium/dependencies |
|
||||||
|
| Search executor/verifier | about 1.7–2.5 GiB | Browser base + application; previous 2.13 GB unpacked is a baseline, not a ceiling |
|
||||||
|
| Engine | roughly 3.5–6 GiB | Restored shared base + Joern; the supplied 2.16 GB compressed engine cannot establish its unpacked size |
|
||||||
|
| Replay | engine + a few KiB | Same engine digest plus replay entrypoint |
|
||||||
|
| VM/kind openultrasast | roughly 3.5–6 GiB plus benchmark assets | Same base/Joern, runtime extras shared, benchmark payload retained |
|
||||||
|
|
||||||
|
An engine + search rotation therefore sums to about **5.2–8.5 GiB** before
|
||||||
|
rollback images. Adding replay as another unpacked image brings the sum to about
|
||||||
|
**8.7–14.5 GiB**; this rotation is not demonstrated to fit 9 GiB. Engine/replay
|
||||||
|
share almost all layers, but the operator must confirm the accounting and measured
|
||||||
|
footprint before selecting a rotation. Extra revisions and the VM image add to
|
||||||
|
the budget if retained there.
|
||||||
|
|
||||||
|
The previous search history included a 1.37 GB apt layer and 281 MB JDK; no
|
||||||
|
per-package apt breakdown is available. Removing system Gradle and docs/caches
|
||||||
|
may save space, while retained build tools and shared Python extras cost space.
|
||||||
|
These ranges are not measured savings or a capacity guarantee. After publication,
|
||||||
|
measure unpacked images and unique layers for the entire intended rotation, and
|
||||||
|
verify build recipes, PHP/JavaScript retention, runner startup and Chromium before
|
||||||
|
rollout. There is no separate <1.5 GiB executor gate.
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# syntax=docker/dockerfile:1
|
||||||
|
# base-browser is built against the published base digest in CI; local builds
|
||||||
|
# default to the base stage. Never independently rebuild its shared ancestry.
|
||||||
|
ARG TASK_BASE_REF=base
|
||||||
|
FROM eclipse-temurin:21-jdk-jammy AS jdk
|
||||||
|
|
||||||
|
FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS base
|
||||||
|
LABEL org.opencontainers.image.licenses="Apache-2.0" \
|
||||||
|
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
|
||||||
|
USER root
|
||||||
|
COPY --from=jdk /opt/java/openjdk /opt/java/openjdk
|
||||||
|
ENV JAVA_HOME=/opt/java/openjdk
|
||||||
|
ENV PATH="/venv/bin:/opt/java/openjdk/bin:${PATH}"
|
||||||
|
# Use upstream /usr/local Python 3.12 (also required by frontend retention).
|
||||||
|
# Its venv/pip and the toolchains below support every executor build recipe.
|
||||||
|
# Gradle projects use their wrapper; no system Gradle is installed.
|
||||||
|
RUN printf 'path-exclude=/usr/share/doc/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/info/*\n' \
|
||||||
|
> /etc/dpkg/dpkg.cfg.d/01-no-docs \
|
||||||
|
&& apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
nodejs npm node-typescript php-cli composer maven sqlite3 bubblewrap util-linux git ca-certificates libstdc++6 zlib1g \
|
||||||
|
&& apt-get clean \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* /usr/share/doc/* /usr/share/man/* /usr/share/info/* \
|
||||||
|
/opt/java/openjdk/lib/src.zip /opt/java/openjdk/man /root/.cache
|
||||||
|
# Metadata only: install runtime dependencies, including the VM lane's extras.
|
||||||
|
# No project package or packaging backend enters this shared environment.
|
||||||
|
COPY pyproject.toml /tmp/dependencies/pyproject.toml
|
||||||
|
RUN python3 -c 'import pathlib,tomllib; p=tomllib.loads(pathlib.Path("/tmp/dependencies/pyproject.toml").read_text())["project"]; print("\n".join(p["dependencies"] + p["optional-dependencies"]["semantic"] + p["optional-dependencies"]["s3"]))' > /tmp/requirements.txt \
|
||||||
|
&& python3 -m venv /venv \
|
||||||
|
&& /venv/bin/pip install --no-cache-dir -r /tmp/requirements.txt \
|
||||||
|
&& rm -rf /tmp/dependencies /tmp/requirements.txt /root/.cache \
|
||||||
|
&& python3 --version && java -version && javac -version && node --version && npm --version && tsc --version \
|
||||||
|
&& php --version && composer --version && mvn --version && sqlite3 --version && bwrap --version \
|
||||||
|
&& setpriv --version && prlimit --version && nsenter --version
|
||||||
|
|
||||||
|
FROM ${TASK_BASE_REF} AS base-browser
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends chromium \
|
||||||
|
&& apt-get clean \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* /usr/share/doc/* /usr/share/man/* /usr/share/info/* /root/.cache \
|
||||||
|
&& chromium --version
|
||||||
@@ -1,23 +1,15 @@
|
|||||||
# Build from the repository root. AX v0.3.1 starts spec.command immediately;
|
# syntax=docker/dockerfile:1
|
||||||
# preserve its runner as PID 1 and put the engine command in the Task manifest.
|
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
|
||||||
FROM eclipse-temurin:21-jre-jammy AS jre
|
ARG TASK_BASE_TAG=base-unpublished
|
||||||
|
ARG TASK_BASE_DIGEST
|
||||||
FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS engine
|
ARG ENGINE_REF=engine
|
||||||
LABEL org.opencontainers.image.licenses="Apache-2.0" \
|
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS joern-build
|
||||||
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
|
ENV PATH="/opt/joern-cli:${PATH}"
|
||||||
|
|
||||||
USER root
|
|
||||||
ARG JOERN_VERSION=v4.0.625
|
ARG JOERN_VERSION=v4.0.625
|
||||||
ARG ARCHIVE=joern-cli-linux-x86_64.zip
|
ARG ARCHIVE=joern-cli-linux-x86_64.zip
|
||||||
|
|
||||||
# The AX runner is Python 3.12 / Debian slim. Copy the Java 21 runtime from
|
|
||||||
# Jammy (older glibc than Debian bookworm) instead of relying on distro JRE versions.
|
|
||||||
COPY --from=jre /opt/java/openjdk /opt/java/openjdk
|
|
||||||
ENV JAVA_HOME=/opt/java/openjdk
|
|
||||||
ENV PATH="/venv/bin:/opt/java/openjdk/bin:/opt/joern-cli:${PATH}"
|
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends \
|
&& apt-get install -y --no-install-recommends curl unzip \
|
||||||
php-cli unzip curl ca-certificates libstdc++6 zlib1g \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN set -eux; \
|
RUN set -eux; \
|
||||||
@@ -37,24 +29,20 @@ COPY ops/frontend-retention/install.py /tmp/frontend-retention-install.py
|
|||||||
RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \
|
RUN python3 /tmp/frontend-retention-install.py /opt/joern-cli \
|
||||||
&& rm /tmp/frontend-retention-install.py
|
&& rm /tmp/frontend-retention-install.py
|
||||||
|
|
||||||
|
FROM ghcr.io/norandom/ousast-task-base:${TASK_BASE_TAG}@${TASK_BASE_DIGEST} AS engine
|
||||||
|
ENV PATH="/opt/joern-cli:${PATH}"
|
||||||
|
COPY --from=joern-build /opt/joern-cli /opt/joern-cli
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY pyproject.toml README.md LICENSE ./
|
COPY pyproject.toml README.md LICENSE ./
|
||||||
COPY src ./src
|
COPY src ./src
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||||
|
/venv/bin/pip install --no-deps --no-cache-dir .
|
||||||
COPY benchmarks/learn/engine_task_entry.py \
|
COPY benchmarks/learn/engine_task_entry.py \
|
||||||
benchmarks/learn/engine_trace_worker.py \
|
benchmarks/learn/engine_trace_worker.py \
|
||||||
benchmarks/learn/engine_trace_parse.py ./benchmarks/learn/
|
benchmarks/learn/engine_trace_parse.py ./benchmarks/learn/
|
||||||
# Worker imports are standard-library plus engine modules. Isolate them from
|
RUN /venv/bin/python -I -c 'import pathlib, runpy, sys; sys.modules.update(yaml=None, boto3=None); sys.path.insert(0, "/app/benchmarks/learn"); p = pathlib.Path("/app/benchmarks/learn/engine_trace_worker.py"); print("Worker import smoke bytes:", len(p.read_bytes())); runpy.run_path(str(p), run_name="engine_worker_import_smoke")'
|
||||||
# the AX runner's Python environment and omit PyYAML, boto3 and all extras.
|
# ENTRYPOINT and CMD belong to the upstream AX runner.
|
||||||
RUN python3 -m venv /venv \
|
|
||||||
&& /venv/bin/pip install --no-cache-dir --no-deps . \
|
|
||||||
&& /venv/bin/python -I -c 'import pathlib, runpy, sys; sys.modules.update(yaml=None, boto3=None); sys.path.insert(0, "/app/benchmarks/learn"); p = pathlib.Path("/app/benchmarks/learn/engine_trace_worker.py"); print("Worker import smoke bytes:", len(p.read_bytes())); runpy.run_path(str(p), run_name="engine_worker_import_smoke")'
|
|
||||||
|
|
||||||
# Do not set ENTRYPOINT or CMD: both belong to the upstream AX runner.
|
FROM ${ENGINE_REF} AS replay
|
||||||
|
|
||||||
FROM engine AS replay
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends git \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
COPY benchmarks/unseen/replay_entry.py ./benchmarks/unseen/replay_entry.py
|
COPY benchmarks/unseen/replay_entry.py ./benchmarks/unseen/replay_entry.py
|
||||||
RUN /venv/bin/pip install --no-cache-dir . \
|
RUN /venv/bin/ousast pre-push --help > /dev/null
|
||||||
&& /venv/bin/ousast pre-push --help > /dev/null
|
|
||||||
|
|||||||
@@ -1,35 +1,19 @@
|
|||||||
# Build from the repository root; retain the AX runner entrypoint and command.
|
# syntax=docker/dockerfile:1
|
||||||
FROM eclipse-temurin:17-jdk-jammy AS jdk
|
# Supply digests from plane/task-base.digest; deliberately no floating fallback.
|
||||||
FROM ghcr.io/norandom/ax-task-runner:v0.3.1
|
ARG TASK_BROWSER_TAG=base-unpublished-browser
|
||||||
LABEL org.opencontainers.image.licenses="Apache-2.0" \
|
ARG TASK_BROWSER_DIGEST
|
||||||
org.opencontainers.image.source="https://github.com/norandom/OpenUltraSAST"
|
FROM ghcr.io/norandom/ousast-task-base:${TASK_BROWSER_TAG}@${TASK_BROWSER_DIGEST}
|
||||||
USER root
|
USER root
|
||||||
COPY --from=jdk /opt/java/openjdk /opt/java/openjdk
|
|
||||||
ENV JAVA_HOME=/opt/java/openjdk
|
|
||||||
ENV PATH="/venv/bin:/opt/java/openjdk/bin:${PATH}"
|
|
||||||
# dpkg exclusions prevent docs/manpages entering layers, including transitive deps.
|
|
||||||
RUN printf 'path-exclude=/usr/share/doc/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/info/*\n' \
|
|
||||||
> /etc/dpkg/dpkg.cfg.d/01-no-docs \
|
|
||||||
&& apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends \
|
|
||||||
python3 python3-venv python3-pip nodejs npm node-typescript php-cli composer maven gradle \
|
|
||||||
sqlite3 bubblewrap util-linux chromium git ca-certificates \
|
|
||||||
&& rm -rf /var/lib/apt/lists/* /opt/java/openjdk/lib/src.zip /opt/java/openjdk/man
|
|
||||||
WORKDIR /app
|
|
||||||
COPY pyproject.toml README.md LICENSE ./
|
|
||||||
COPY src ./src
|
|
||||||
# Includes the verifier-owned recipes, oracles, and key-free executor entrypoints.
|
|
||||||
RUN python3 -m venv /venv \
|
|
||||||
&& /venv/bin/pip install --no-cache-dir . \
|
|
||||||
&& python3 --version && node --version && tsc --version && php --version \
|
|
||||||
&& javac -version && sqlite3 --version && bwrap --version \
|
|
||||||
&& setpriv --version && prlimit --version && nsenter --version && chromium --version
|
|
||||||
# Dedicated identity for artefacts, apps and Chromium; the verifier keeps root.
|
# Dedicated identity for artefacts, apps and Chromium; the verifier keeps root.
|
||||||
RUN groupadd --gid 10001 search \
|
RUN groupadd --gid 10001 search \
|
||||||
&& useradd --create-home --uid 10001 --gid 10001 search
|
&& useradd --create-home --uid 10001 --gid 10001 search
|
||||||
# gVisor is the outer boundary; retain inner bwrap when its capability probe succeeds.
|
# Upstream's public CPython signing fingerprint trips the credential guard.
|
||||||
# The Python base image sets GPG_KEY to the public CPython release-signing fingerprint (not a secret).
|
|
||||||
# Clear it so the executor's guard against inherited credentials stays strict and still passes.
|
|
||||||
ENV GPG_KEY=
|
ENV GPG_KEY=
|
||||||
|
WORKDIR /app
|
||||||
|
COPY pyproject.toml README.md LICENSE ./
|
||||||
|
COPY src ./src
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||||
|
/venv/bin/pip install --no-deps --no-cache-dir .
|
||||||
|
# Entrypoints and verifier-owned recipes/oracles are included in src.
|
||||||
USER root
|
USER root
|
||||||
# Do not replace the AX runner's ENTRYPOINT or CMD.
|
# Preserve the AX runner ENTRYPOINT and CMD.
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"base": null,
|
||||||
|
"base-browser": null,
|
||||||
|
"note": "UNPUBLISHED: run task-base-image, then replace with the two full tag@sha256 references from its summary. Never invent digests."
|
||||||
|
}
|
||||||
@@ -274,9 +274,31 @@ class InProcessExecutor:
|
|||||||
}
|
}
|
||||||
|
|
||||||
def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]:
|
def _task_run(self, argv: list[str], timeout: float, limits: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
process_limit = None
|
||||||
|
try:
|
||||||
|
uid = os.getuid()
|
||||||
|
current = 0
|
||||||
|
for entry in Path("/proc").iterdir():
|
||||||
|
if not entry.name.isdecimal():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
status = (entry / "status").read_text()
|
||||||
|
except FileNotFoundError:
|
||||||
|
continue
|
||||||
|
fields = dict(line.split(":", 1) for line in status.splitlines() if ":" in line)
|
||||||
|
# The kernel's NPROC counter counts threads, not processes: a single IDE or agent
|
||||||
|
# process with 60 threads uses 60 of the allowance.
|
||||||
|
if int(fields["Uid"].split()[0]) == uid:
|
||||||
|
current += int(fields.get("Threads", "1").strip() or 1)
|
||||||
|
process_limit = current + 256
|
||||||
|
except (OSError, ValueError, IndexError, StopIteration):
|
||||||
|
pass
|
||||||
|
|
||||||
def constrain() -> None:
|
def constrain() -> None:
|
||||||
resource.setrlimit(resource.RLIMIT_FSIZE, (limits.get("disk_bytes", task_storage.DEFAULT_SCRATCH_BYTES),) * 2)
|
resource.setrlimit(resource.RLIMIT_FSIZE, (limits.get("disk_bytes", task_storage.DEFAULT_SCRATCH_BYTES),) * 2)
|
||||||
resource.setrlimit(resource.RLIMIT_NPROC, (128, 128))
|
# NPROC counts all processes of the real uid, so an absolute 128 blocks builds on busy hosts.
|
||||||
|
if process_limit is not None:
|
||||||
|
resource.setrlimit(resource.RLIMIT_NPROC, (process_limit,) * 2)
|
||||||
resource.setrlimit(resource.RLIMIT_AS, (limits.get("memory_bytes", 256 * 1024**2),) * 2)
|
resource.setrlimit(resource.RLIMIT_AS, (limits.get("memory_bytes", 256 * 1024**2),) * 2)
|
||||||
resource.setrlimit(resource.RLIMIT_CPU, (math.ceil(timeout) + 1,) * 2)
|
resource.setrlimit(resource.RLIMIT_CPU, (math.ceil(timeout) + 1,) * 2)
|
||||||
|
|
||||||
@@ -391,7 +413,10 @@ class InProcessExecutor:
|
|||||||
if recipe["recipe"] == "maven":
|
if recipe["recipe"] == "maven":
|
||||||
argv += ["package", "-DskipTests"]
|
argv += ["package", "-DskipTests"]
|
||||||
elif recipe["recipe"] == "gradle":
|
elif recipe["recipe"] == "gradle":
|
||||||
argv += ["assemble"]
|
# The project pins its Gradle version through its wrapper.
|
||||||
|
# Invoke with sh because checkout archives may lose execute bits.
|
||||||
|
wrapper = self._path(str(path.relative_to(self.repo) / "gradlew"))
|
||||||
|
argv = ["/bin/sh", str(wrapper), "--no-daemon", "--project-dir", str(path), "assemble"]
|
||||||
built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES})
|
built = self._task_run(argv, self._remaining(), {"disk_bytes": task_storage.DEFAULT_SCRATCH_BYTES})
|
||||||
if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
|
if built.get("disk_limit_exceeded") or built["exit_code"] or built["timed_out"]:
|
||||||
return {**built, "status": "could_not_build"}
|
return {**built, "status": "could_not_build"}
|
||||||
|
|||||||
@@ -4,17 +4,17 @@ import yaml
|
|||||||
|
|
||||||
|
|
||||||
def test_image_matches_runner_and_publishes_digest():
|
def test_image_matches_runner_and_publishes_digest():
|
||||||
engine = Path("plane/Dockerfile.engine-task").read_text()
|
base = Path("plane/Dockerfile.base").read_text()
|
||||||
image = Path("plane/Dockerfile.search-task").read_text()
|
image = Path("plane/Dockerfile.search-task").read_text()
|
||||||
runner = "FROM ghcr.io/norandom/ax-task-runner:v0.3.1"
|
assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1" in base
|
||||||
assert runner in engine and runner in image
|
for package in ("nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"):
|
||||||
for package in ("python3", "nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"):
|
assert package in base
|
||||||
assert package in image
|
assert "python3 -m venv /venv" in base
|
||||||
assert image.rsplit("USER ", 1)[1].startswith("root\n")
|
assert image.rsplit("USER ", 1)[1].startswith("root\n")
|
||||||
assert "--uid 10001 --gid 10001 search" in image
|
assert "--uid 10001 --gid 10001 search" in image
|
||||||
assert "setpriv --version" in image
|
assert "setpriv --version" in base
|
||||||
assert "17-jdk" in image and "--no-install-recommends" in image
|
assert "21-jdk" in base and "--no-install-recommends" in base
|
||||||
assert "rm -rf /var/lib/apt/lists/" in image
|
assert "rm -rf /var/lib/apt/lists/" in base
|
||||||
workflow_text = Path(".github/workflows/engine-image.yml").read_text()
|
workflow_text = Path(".github/workflows/engine-image.yml").read_text()
|
||||||
entries = yaml.safe_load(workflow_text)["jobs"]["image"]["strategy"]["matrix"]["include"]
|
entries = yaml.safe_load(workflow_text)["jobs"]["image"]["strategy"]["matrix"]["include"]
|
||||||
assert {"package": "ousast-search-task", "dockerfile": "plane/Dockerfile.search-task"} in entries
|
assert {"package": "ousast-search-task", "dockerfile": "plane/Dockerfile.search-task"} in entries
|
||||||
|
|||||||
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import signal
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -224,19 +226,58 @@ def test_scratch_failure_preserves_exit_and_stderr(workspace, monkeypatch):
|
|||||||
assert result["reason"].startswith("ScratchLimit:")
|
assert result["reason"].startswith("ScratchLimit:")
|
||||||
|
|
||||||
|
|
||||||
def test_capacity_deadline_plain_diagnostic_survives():
|
@pytest.mark.parametrize("project", [".", "subproject"])
|
||||||
from benchmarks.ax.batch import CapacityDeadline
|
def test_preparation_uses_project_gradle_wrapper(workspace, monkeypatch, project):
|
||||||
|
repo = workspace / "checkout"
|
||||||
message = "no free workers available before deadline"
|
repo.mkdir()
|
||||||
assert task_storage.diagnostic(message) == message
|
root = repo / project
|
||||||
assert task_storage.exception_reason(CapacityDeadline(message)) == "CapacityDeadline: " + message
|
root.mkdir(exist_ok=True)
|
||||||
|
# A real local child proves wrapper execution without system Gradle or network.
|
||||||
|
wrapper = root / "gradlew"
|
||||||
|
wrapper.write_text(
|
||||||
|
'test "$1" = "--no-daemon" || exit 1\n'
|
||||||
|
'test "$2" = "--project-dir" || exit 2\n'
|
||||||
|
'test "$4" = "assemble" || exit 3\n'
|
||||||
|
'mkdir -p "$3/build"\n'
|
||||||
|
'printf compiled > "$3/build/product"\n'
|
||||||
|
)
|
||||||
|
wrapper.chmod(0o644)
|
||||||
|
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||||
|
app.prepared_put_url = "https://files.example/output"
|
||||||
|
blobs = []
|
||||||
|
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
|
||||||
|
request = spec()
|
||||||
|
request["demo"]["build"] = {"recipe": "gradle", "arguments": [project]}
|
||||||
|
result = app.submit("prepare_verification", request)
|
||||||
|
assert result["status"] == "ok", result
|
||||||
|
unpacked = workspace / "unpacked"
|
||||||
|
verify_task.extract_checkout(blobs[0], unpacked)
|
||||||
|
assert (unpacked / "checkout" / project / "build/product").read_text() == "compiled"
|
||||||
|
assert json.loads((unpacked / "spec.json").read_text())["demo"]["build"]["recipe"] == "none"
|
||||||
|
|
||||||
|
|
||||||
def test_cleanup_diagnostic_retains_both_causes():
|
def test_preparation_with_many_host_processes(workspace, monkeypatch):
|
||||||
error = ValueError("original apply failed")
|
repo = workspace / "checkout"
|
||||||
error.executor_cleanup_reason = "executor cleanup failed: " + "details " * 100 + "delete failed"
|
repo.mkdir()
|
||||||
error.args = (str(error) + "; " + error.executor_cleanup_reason,)
|
(repo / "gradlew").write_text('mkdir -p "$3/build"\nprintf compiled > "$3/build/product"\n')
|
||||||
reason = task_storage.exception_reason(error)
|
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||||
assert reason.startswith("ValueError: original apply failed;")
|
app.prepared_put_url = "https://files.example/output"
|
||||||
assert reason.endswith("delete failed")
|
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: None)
|
||||||
assert len(reason) <= 300
|
request = spec()
|
||||||
|
request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]}
|
||||||
|
children = []
|
||||||
|
try:
|
||||||
|
for _ in range(300):
|
||||||
|
pid = os.fork()
|
||||||
|
if pid == 0:
|
||||||
|
time.sleep(60)
|
||||||
|
os._exit(0)
|
||||||
|
children.append(pid)
|
||||||
|
result = app.submit("prepare_verification", request)
|
||||||
|
assert result["status"] == "ok", result
|
||||||
|
assert (repo / "build/product").read_text() == "compiled"
|
||||||
|
finally:
|
||||||
|
for pid in children:
|
||||||
|
os.kill(pid, signal.SIGKILL)
|
||||||
|
for pid in children:
|
||||||
|
os.waitpid(pid, 0)
|
||||||
|
|||||||
@@ -0,0 +1,164 @@
|
|||||||
|
"""Offline image contracts; these do not claim a built image or measured size."""
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def read(name):
|
||||||
|
return (ROOT / name).read_text()
|
||||||
|
|
||||||
|
|
||||||
|
def instructions(name):
|
||||||
|
text = re.sub(r"\\\n\s*", " ", read(name))
|
||||||
|
return [line for line in text.splitlines() if line and not line.startswith("#")]
|
||||||
|
|
||||||
|
|
||||||
|
def workflow(name):
|
||||||
|
# BaseLoader avoids YAML 1.1 interpreting GitHub's `on` as a boolean.
|
||||||
|
return yaml.load(read(f".github/workflows/{name}.yml"), Loader=yaml.BaseLoader)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name,variant", [
|
||||||
|
("plane/Dockerfile.engine-task", "BASE"),
|
||||||
|
("plane/Dockerfile.search-task", "BROWSER"),
|
||||||
|
("Dockerfile", "BASE"),
|
||||||
|
])
|
||||||
|
def test_digest_required_and_code_last(name, variant):
|
||||||
|
lines = instructions(name)
|
||||||
|
assert f"ARG TASK_{variant}_DIGEST" in lines
|
||||||
|
ref = f"ghcr.io/norandom/ousast-task-base:${{TASK_{variant}_TAG}}@${{TASK_{variant}_DIGEST}}"
|
||||||
|
assert any(line.startswith(f"FROM {ref}") for line in lines)
|
||||||
|
code = next(i for i, line in enumerate(lines) if line.startswith("COPY src "))
|
||||||
|
for i, line in enumerate(lines):
|
||||||
|
if "apt-get" in line or "--from=joern-build" in line or "frontend-retention-install.py" in line:
|
||||||
|
assert i < code
|
||||||
|
if "pip install" in line:
|
||||||
|
assert "--no-deps --no-cache-dir ." in line
|
||||||
|
assert i > code
|
||||||
|
assert not any("venv /venv" in line for line in lines)
|
||||||
|
|
||||||
|
|
||||||
|
def test_shared_ancestry_and_replay_exact_engine():
|
||||||
|
base = read("plane/Dockerfile.base")
|
||||||
|
assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS base" in base
|
||||||
|
assert "FROM ${TASK_BASE_REF} AS base-browser" in base
|
||||||
|
common, browser = base.split("FROM ${TASK_BASE_REF} AS base-browser")
|
||||||
|
assert "chromium" not in common and "chromium" in browser
|
||||||
|
assert "21-jdk-jammy" in common
|
||||||
|
assert "COPY src" not in base and 'p["dependencies"]' in base
|
||||||
|
assert "pip install --no-cache-dir -r /tmp/requirements.txt" in base
|
||||||
|
engine = read("plane/Dockerfile.engine-task")
|
||||||
|
assert "ARG ENGINE_REF=engine" in engine and "FROM ${ENGINE_REF} AS replay" in engine
|
||||||
|
wf = workflow("engine-image")
|
||||||
|
assert wf["jobs"]["replay"]["needs"] == ["pins", "engine"]
|
||||||
|
builds = [s["with"] for s in wf["jobs"]["replay"]["steps"] if s.get("uses") == "docker/build-push-action@v6"]
|
||||||
|
assert "ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }}" in builds[0]["build-args"]
|
||||||
|
base_wf = workflow("task-base-image")
|
||||||
|
browser_build = next(s["with"] for s in base_wf["jobs"]["base"]["steps"] if s.get("id") == "browser")
|
||||||
|
assert "@${{ steps.base.outputs.digest }}" in browser_build["build-args"]
|
||||||
|
# Remote ancestry and anonymous existence are checked before downstream builds.
|
||||||
|
assert 'manifests["base-browser"][:len(base)] != base' in read(".github/workflows/engine-image.yml")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name", ["engine-image", "task-base-image"])
|
||||||
|
def test_triggers_caches_and_digest_outputs(name):
|
||||||
|
wf = workflow(name)
|
||||||
|
assert "workflow_dispatch" in wf["on"]
|
||||||
|
assert "src/**" not in wf["on"]["push"]["paths"]
|
||||||
|
if name == "engine-image":
|
||||||
|
assert wf["on"]["push"]["tags"] == ["v*"]
|
||||||
|
assert "plane/task-base.digest" in wf["on"]["push"]["paths"]
|
||||||
|
else:
|
||||||
|
assert set(wf["on"]["push"]["paths"]) == {"plane/Dockerfile.base", "pyproject.toml", "uv.lock"}
|
||||||
|
builds = [s["with"] for job in wf["jobs"].values() for s in job["steps"]
|
||||||
|
if s.get("uses") == "docker/build-push-action@v6"]
|
||||||
|
assert builds
|
||||||
|
for build in builds:
|
||||||
|
assert "type=gha" in build["cache-from"] and "type=gha" in build["cache-to"]
|
||||||
|
assert build["push"] == "true"
|
||||||
|
assert "outputs.digest" in read(f".github/workflows/{name}.yml")
|
||||||
|
|
||||||
|
|
||||||
|
def test_pin_bootstrap_fails_closed():
|
||||||
|
pins = json.loads(read("plane/task-base.digest"))
|
||||||
|
assert set(pins) >= {"base", "base-browser"}
|
||||||
|
for key in ("base", "base-browser"):
|
||||||
|
if pins[key] is None:
|
||||||
|
assert "UNPUBLISHED" in pins["note"]
|
||||||
|
else:
|
||||||
|
assert re.fullmatch(r"ghcr.io/norandom/ousast-task-base:base-[a-z0-9-]+@sha256:[0-9a-f]{64}", pins[key])
|
||||||
|
wf = read(".github/workflows/engine-image.yml")
|
||||||
|
assert "raise SystemExit" in wf and '"inspect", "--raw", ref' in wf
|
||||||
|
assert "needs: pins" in wf
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("case", ["unpublished", "missing", "wrong-ancestry", "valid"])
|
||||||
|
def test_ci_pin_gate_offline(case, tmp_path, monkeypatch):
|
||||||
|
"""Exercise the actual workflow Python, with only registry reads replaced."""
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
pin_step = next(s for s in workflow("engine-image")["jobs"]["pins"]["steps"] if s.get("id") == "pins")
|
||||||
|
script = pin_step["run"].split("<<'PYTHON'\n", 1)[1].rsplit("PYTHON", 1)[0]
|
||||||
|
base = "ghcr.io/norandom/ousast-task-base:base-1234567@sha256:" + "a" * 64
|
||||||
|
browser = "ghcr.io/norandom/ousast-task-base:base-1234567-browser@sha256:" + "b" * 64
|
||||||
|
(tmp_path / "plane").mkdir()
|
||||||
|
(tmp_path / "plane/task-base.digest").write_text(json.dumps({
|
||||||
|
"base": None if case == "unpublished" else base, "base-browser": browser,
|
||||||
|
}))
|
||||||
|
output = tmp_path / "output"
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
monkeypatch.setenv("GITHUB_OUTPUT", str(output))
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def inspect(argv):
|
||||||
|
calls.append(argv)
|
||||||
|
assert argv[:5] == ["docker", "buildx", "imagetools", "inspect", "--raw"]
|
||||||
|
if case == "missing":
|
||||||
|
raise subprocess.CalledProcessError(1, argv)
|
||||||
|
layers = [{"digest": "sha256:shared"}]
|
||||||
|
if argv[-1] == browser:
|
||||||
|
if case == "wrong-ancestry":
|
||||||
|
layers = [{"digest": "sha256:different"}]
|
||||||
|
layers.append({"digest": "sha256:chromium"})
|
||||||
|
return json.dumps({"layers": layers}).encode()
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "check_output", inspect)
|
||||||
|
if case == "valid":
|
||||||
|
exec(compile(script, "workflow-pin-gate", "exec"), {})
|
||||||
|
assert "base_digest=sha256:" in output.read_text()
|
||||||
|
assert "browser_digest=sha256:" in output.read_text()
|
||||||
|
assert len(calls) == 2
|
||||||
|
else:
|
||||||
|
expected = subprocess.CalledProcessError if case == "missing" else SystemExit
|
||||||
|
with pytest.raises(expected):
|
||||||
|
exec(compile(script, "workflow-pin-gate", "exec"), {})
|
||||||
|
if case == "unpublished":
|
||||||
|
assert not calls
|
||||||
|
|
||||||
|
|
||||||
|
def test_base_preserves_executor_build_toolchains():
|
||||||
|
common = read("plane/Dockerfile.base").split("FROM ${TASK_BASE_REF} AS base-browser")[0]
|
||||||
|
commands = " ".join(instructions("plane/Dockerfile.base")).split("FROM ${TASK_BASE_REF} AS base-browser")[0]
|
||||||
|
install = re.search(r"apt-get install -y --no-install-recommends\s+(.*?)\s+&&", commands).group(1)
|
||||||
|
packages = set(install.split())
|
||||||
|
assert packages == {
|
||||||
|
"nodejs", "npm", "node-typescript", "php-cli", "composer", "maven",
|
||||||
|
"sqlite3", "bubblewrap", "util-linux", "git", "ca-certificates",
|
||||||
|
"libstdc++6", "zlib1g",
|
||||||
|
}
|
||||||
|
assert "gradle" not in packages
|
||||||
|
assert "FROM eclipse-temurin:21-jdk-jammy AS jdk" in common
|
||||||
|
assert "COPY --from=jdk /opt/java/openjdk /opt/java/openjdk" in common
|
||||||
|
for command in ("javac -version", "npm --version", "composer --version", "mvn --version",
|
||||||
|
"python3 -m venv /venv", "/venv/bin/pip install"):
|
||||||
|
assert command in common
|
||||||
|
assert "joern" not in common.lower()
|
||||||
|
assert "apt-get clean" in common
|
||||||
|
for directory in ("doc", "man", "info"):
|
||||||
|
assert f"path-exclude=/usr/share/{directory}/*" in common
|
||||||
|
assert f" /usr/share/{directory}/*" in common
|
||||||
Reference in New Issue
Block a user