mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
images: shared task base, code-last layers, no per-push rebuilds [skip ci]
One base (Temurin 21 JDK, node/npm/tsc, php/composer, maven, sqlite, bwrap) built only when its Dockerfile or dependencies change, plus a browser variant; engine/search/VM images FROM base@sha256 with the code in the last layer; the image workflow runs on dispatch, tags, Dockerfile or pin changes, with a build cache. Executor NPROC bound relative to the current process count. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
7fb061fb69
commit
1b9d1ca85c
@@ -4,17 +4,17 @@ import yaml
|
||||
|
||||
|
||||
def test_image_matches_runner_and_publishes_digest():
|
||||
engine = Path("plane/Dockerfile.engine-task").read_text()
|
||||
base = Path("plane/Dockerfile.base").read_text()
|
||||
image = Path("plane/Dockerfile.search-task").read_text()
|
||||
runner = "FROM ghcr.io/norandom/ax-task-runner:v0.3.1"
|
||||
assert runner in engine and runner in image
|
||||
for package in ("python3", "nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"):
|
||||
assert package in image
|
||||
assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1" in base
|
||||
for package in ("nodejs", "node-typescript", "php-cli", "sqlite3", "bubblewrap", "util-linux", "chromium"):
|
||||
assert package in base
|
||||
assert "python3 -m venv /venv" in base
|
||||
assert image.rsplit("USER ", 1)[1].startswith("root\n")
|
||||
assert "--uid 10001 --gid 10001 search" in image
|
||||
assert "setpriv --version" in image
|
||||
assert "17-jdk" in image and "--no-install-recommends" in image
|
||||
assert "rm -rf /var/lib/apt/lists/" in image
|
||||
assert "setpriv --version" in base
|
||||
assert "21-jdk" in base and "--no-install-recommends" in base
|
||||
assert "rm -rf /var/lib/apt/lists/" in base
|
||||
workflow_text = Path(".github/workflows/engine-image.yml").read_text()
|
||||
entries = yaml.safe_load(workflow_text)["jobs"]["image"]["strategy"]["matrix"]["include"]
|
||||
assert {"package": "ousast-search-task", "dockerfile": "plane/Dockerfile.search-task"} in entries
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -222,3 +224,60 @@ def test_scratch_failure_preserves_exit_and_stderr(workspace, monkeypatch):
|
||||
assert result["exit_code"] is not None
|
||||
assert result["stderr"] == "pip build detail"
|
||||
assert result["reason"].startswith("ScratchLimit:")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("project", [".", "subproject"])
|
||||
def test_preparation_uses_project_gradle_wrapper(workspace, monkeypatch, project):
|
||||
repo = workspace / "checkout"
|
||||
repo.mkdir()
|
||||
root = repo / project
|
||||
root.mkdir(exist_ok=True)
|
||||
# A real local child proves wrapper execution without system Gradle or network.
|
||||
wrapper = root / "gradlew"
|
||||
wrapper.write_text(
|
||||
'test "$1" = "--no-daemon" || exit 1\n'
|
||||
'test "$2" = "--project-dir" || exit 2\n'
|
||||
'test "$4" = "assemble" || exit 3\n'
|
||||
'mkdir -p "$3/build"\n'
|
||||
'printf compiled > "$3/build/product"\n'
|
||||
)
|
||||
wrapper.chmod(0o644)
|
||||
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||
app.prepared_put_url = "https://files.example/output"
|
||||
blobs = []
|
||||
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
|
||||
request = spec()
|
||||
request["demo"]["build"] = {"recipe": "gradle", "arguments": [project]}
|
||||
result = app.submit("prepare_verification", request)
|
||||
assert result["status"] == "ok", result
|
||||
unpacked = workspace / "unpacked"
|
||||
verify_task.extract_checkout(blobs[0], unpacked)
|
||||
assert (unpacked / "checkout" / project / "build/product").read_text() == "compiled"
|
||||
assert json.loads((unpacked / "spec.json").read_text())["demo"]["build"]["recipe"] == "none"
|
||||
|
||||
|
||||
def test_preparation_with_many_host_processes(workspace, monkeypatch):
|
||||
repo = workspace / "checkout"
|
||||
repo.mkdir()
|
||||
(repo / "gradlew").write_text('mkdir -p "$3/build"\nprintf compiled > "$3/build/product"\n')
|
||||
app = executor.InProcessExecutor(repo, task_boundary=True)
|
||||
app.prepared_put_url = "https://files.example/output"
|
||||
monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: None)
|
||||
request = spec()
|
||||
request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]}
|
||||
children = []
|
||||
try:
|
||||
for _ in range(300):
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
time.sleep(60)
|
||||
os._exit(0)
|
||||
children.append(pid)
|
||||
result = app.submit("prepare_verification", request)
|
||||
assert result["status"] == "ok", result
|
||||
assert (repo / "build/product").read_text() == "compiled"
|
||||
finally:
|
||||
for pid in children:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
for pid in children:
|
||||
os.waitpid(pid, 0)
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Offline image contracts; these do not claim a built image or measured size."""
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def read(name):
|
||||
return (ROOT / name).read_text()
|
||||
|
||||
|
||||
def instructions(name):
|
||||
text = re.sub(r"\\\n\s*", " ", read(name))
|
||||
return [line for line in text.splitlines() if line and not line.startswith("#")]
|
||||
|
||||
|
||||
def workflow(name):
|
||||
# BaseLoader avoids YAML 1.1 interpreting GitHub's `on` as a boolean.
|
||||
return yaml.load(read(f".github/workflows/{name}.yml"), Loader=yaml.BaseLoader)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,variant", [
|
||||
("plane/Dockerfile.engine-task", "BASE"),
|
||||
("plane/Dockerfile.search-task", "BROWSER"),
|
||||
("Dockerfile", "BASE"),
|
||||
])
|
||||
def test_digest_required_and_code_last(name, variant):
|
||||
lines = instructions(name)
|
||||
assert f"ARG TASK_{variant}_DIGEST" in lines
|
||||
ref = f"ghcr.io/norandom/ousast-task-base:${{TASK_{variant}_TAG}}@${{TASK_{variant}_DIGEST}}"
|
||||
assert any(line.startswith(f"FROM {ref}") for line in lines)
|
||||
code = next(i for i, line in enumerate(lines) if line.startswith("COPY src "))
|
||||
for i, line in enumerate(lines):
|
||||
if "apt-get" in line or "--from=joern-build" in line or "frontend-retention-install.py" in line:
|
||||
assert i < code
|
||||
if "pip install" in line:
|
||||
assert "--no-deps --no-cache-dir ." in line
|
||||
assert i > code
|
||||
assert not any("venv /venv" in line for line in lines)
|
||||
|
||||
|
||||
def test_shared_ancestry_and_replay_exact_engine():
|
||||
base = read("plane/Dockerfile.base")
|
||||
assert "FROM ghcr.io/norandom/ax-task-runner:v0.3.1 AS base" in base
|
||||
assert "FROM ${TASK_BASE_REF} AS base-browser" in base
|
||||
common, browser = base.split("FROM ${TASK_BASE_REF} AS base-browser")
|
||||
assert "chromium" not in common and "chromium" in browser
|
||||
assert "21-jdk-jammy" in common
|
||||
assert "COPY src" not in base and 'p["dependencies"]' in base
|
||||
assert "pip install --no-cache-dir -r /tmp/requirements.txt" in base
|
||||
engine = read("plane/Dockerfile.engine-task")
|
||||
assert "ARG ENGINE_REF=engine" in engine and "FROM ${ENGINE_REF} AS replay" in engine
|
||||
wf = workflow("engine-image")
|
||||
assert wf["jobs"]["replay"]["needs"] == ["pins", "engine"]
|
||||
builds = [s["with"] for s in wf["jobs"]["replay"]["steps"] if s.get("uses") == "docker/build-push-action@v6"]
|
||||
assert "ENGINE_REF=${{ needs.engine.outputs.image }}@${{ needs.engine.outputs.digest }}" in builds[0]["build-args"]
|
||||
base_wf = workflow("task-base-image")
|
||||
browser_build = next(s["with"] for s in base_wf["jobs"]["base"]["steps"] if s.get("id") == "browser")
|
||||
assert "@${{ steps.base.outputs.digest }}" in browser_build["build-args"]
|
||||
# Remote ancestry and anonymous existence are checked before downstream builds.
|
||||
assert 'manifests["base-browser"][:len(base)] != base' in read(".github/workflows/engine-image.yml")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["engine-image", "task-base-image"])
|
||||
def test_triggers_caches_and_digest_outputs(name):
|
||||
wf = workflow(name)
|
||||
assert "workflow_dispatch" in wf["on"]
|
||||
assert "src/**" not in wf["on"]["push"]["paths"]
|
||||
if name == "engine-image":
|
||||
assert wf["on"]["push"]["tags"] == ["v*"]
|
||||
assert "plane/task-base.digest" in wf["on"]["push"]["paths"]
|
||||
else:
|
||||
assert set(wf["on"]["push"]["paths"]) == {"plane/Dockerfile.base", "pyproject.toml", "uv.lock"}
|
||||
builds = [s["with"] for job in wf["jobs"].values() for s in job["steps"]
|
||||
if s.get("uses") == "docker/build-push-action@v6"]
|
||||
assert builds
|
||||
for build in builds:
|
||||
assert "type=gha" in build["cache-from"] and "type=gha" in build["cache-to"]
|
||||
assert build["push"] == "true"
|
||||
assert "outputs.digest" in read(f".github/workflows/{name}.yml")
|
||||
|
||||
|
||||
def test_pin_bootstrap_fails_closed():
|
||||
pins = json.loads(read("plane/task-base.digest"))
|
||||
assert set(pins) >= {"base", "base-browser"}
|
||||
for key in ("base", "base-browser"):
|
||||
if pins[key] is None:
|
||||
assert "UNPUBLISHED" in pins["note"]
|
||||
else:
|
||||
assert re.fullmatch(r"ghcr.io/norandom/ousast-task-base:base-[a-z0-9-]+@sha256:[0-9a-f]{64}", pins[key])
|
||||
wf = read(".github/workflows/engine-image.yml")
|
||||
assert "raise SystemExit" in wf and '"inspect", "--raw", ref' in wf
|
||||
assert "needs: pins" in wf
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["unpublished", "missing", "wrong-ancestry", "valid"])
|
||||
def test_ci_pin_gate_offline(case, tmp_path, monkeypatch):
|
||||
"""Exercise the actual workflow Python, with only registry reads replaced."""
|
||||
import subprocess
|
||||
|
||||
pin_step = next(s for s in workflow("engine-image")["jobs"]["pins"]["steps"] if s.get("id") == "pins")
|
||||
script = pin_step["run"].split("<<'PYTHON'\n", 1)[1].rsplit("PYTHON", 1)[0]
|
||||
base = "ghcr.io/norandom/ousast-task-base:base-1234567@sha256:" + "a" * 64
|
||||
browser = "ghcr.io/norandom/ousast-task-base:base-1234567-browser@sha256:" + "b" * 64
|
||||
(tmp_path / "plane").mkdir()
|
||||
(tmp_path / "plane/task-base.digest").write_text(json.dumps({
|
||||
"base": None if case == "unpublished" else base, "base-browser": browser,
|
||||
}))
|
||||
output = tmp_path / "output"
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("GITHUB_OUTPUT", str(output))
|
||||
calls = []
|
||||
|
||||
def inspect(argv):
|
||||
calls.append(argv)
|
||||
assert argv[:5] == ["docker", "buildx", "imagetools", "inspect", "--raw"]
|
||||
if case == "missing":
|
||||
raise subprocess.CalledProcessError(1, argv)
|
||||
layers = [{"digest": "sha256:shared"}]
|
||||
if argv[-1] == browser:
|
||||
if case == "wrong-ancestry":
|
||||
layers = [{"digest": "sha256:different"}]
|
||||
layers.append({"digest": "sha256:chromium"})
|
||||
return json.dumps({"layers": layers}).encode()
|
||||
|
||||
monkeypatch.setattr(subprocess, "check_output", inspect)
|
||||
if case == "valid":
|
||||
exec(compile(script, "workflow-pin-gate", "exec"), {})
|
||||
assert "base_digest=sha256:" in output.read_text()
|
||||
assert "browser_digest=sha256:" in output.read_text()
|
||||
assert len(calls) == 2
|
||||
else:
|
||||
expected = subprocess.CalledProcessError if case == "missing" else SystemExit
|
||||
with pytest.raises(expected):
|
||||
exec(compile(script, "workflow-pin-gate", "exec"), {})
|
||||
if case == "unpublished":
|
||||
assert not calls
|
||||
|
||||
|
||||
def test_base_preserves_executor_build_toolchains():
|
||||
common = read("plane/Dockerfile.base").split("FROM ${TASK_BASE_REF} AS base-browser")[0]
|
||||
commands = " ".join(instructions("plane/Dockerfile.base")).split("FROM ${TASK_BASE_REF} AS base-browser")[0]
|
||||
install = re.search(r"apt-get install -y --no-install-recommends\s+(.*?)\s+&&", commands).group(1)
|
||||
packages = set(install.split())
|
||||
assert packages == {
|
||||
"nodejs", "npm", "node-typescript", "php-cli", "composer", "maven",
|
||||
"sqlite3", "bubblewrap", "util-linux", "git", "ca-certificates",
|
||||
"libstdc++6", "zlib1g",
|
||||
}
|
||||
assert "gradle" not in packages
|
||||
assert "FROM eclipse-temurin:21-jdk-jammy AS jdk" in common
|
||||
assert "COPY --from=jdk /opt/java/openjdk /opt/java/openjdk" in common
|
||||
for command in ("javac -version", "npm --version", "composer --version", "mvn --version",
|
||||
"python3 -m venv /venv", "/venv/bin/pip install"):
|
||||
assert command in common
|
||||
assert "joern" not in common.lower()
|
||||
assert "apt-get clean" in common
|
||||
for directory in ("doc", "man", "info"):
|
||||
assert f"path-exclude=/usr/share/{directory}/*" in common
|
||||
assert f" /usr/share/{directory}/*" in common
|
||||
Reference in New Issue
Block a user