mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Fable 5.1
0d3e6df31c
Archives are gzipped and uploaded in parts of at most 90 MB with per-part digests (edge request limit); pip installs ignore the task venv so transitive dependencies land in products; the verifier sets NODE_PATH and the Java classpath from products when starting the app. Co-Authored-By: Claude Fable 5.1 <[email protected]>
295 lines
11 KiB
Python
295 lines
11 KiB
Python
"""No-network end-to-end executor smoke, through the real mailbox client."""
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from benchmarks.ax import batch
|
|
from benchmarks.search import executor_smoke as smoke
|
|
from openultrasast.search.executor import Command, InProcessExecutor
|
|
|
|
|
|
class Store:
|
|
def __init__(self, executor):
|
|
self.executor, self.objects = executor, {}
|
|
|
|
def _put(self, key, data):
|
|
self.objects[key] = data
|
|
|
|
def _delete(self, key):
|
|
self.objects.pop(key, None)
|
|
|
|
def presign_get(self, key, expires):
|
|
return "https://files.example/" + key
|
|
|
|
presign_put = presign_get
|
|
|
|
def put(self, url, data, timeout):
|
|
self.result = json.dumps(self.executor.execute(Command(**json.loads(data)))).encode()
|
|
|
|
def get(self, url, timeout):
|
|
return self.result
|
|
|
|
|
|
class Executor(InProcessExecutor):
|
|
empty = False
|
|
empty_install = False
|
|
|
|
def _task_run(self, argv, timeout, limits):
|
|
if argv[:2] == ["sh", "-ec"]:
|
|
if not self.empty:
|
|
(self.repo / "README.md").write_text("test input")
|
|
(self.repo / "requirements.txt").write_text("")
|
|
return dict(exit_code=0, stdout="cloned", stderr="", scratch_peak_bytes=4096, stdout_bytes=6, stderr_bytes=0)
|
|
if "pip" in argv:
|
|
return dict(
|
|
exit_code=0,
|
|
stdout="" if self.empty_install else "installed",
|
|
stderr="",
|
|
scratch_peak_bytes=8192,
|
|
stdout_bytes=0 if self.empty_install else 9,
|
|
stderr_bytes=0,
|
|
)
|
|
return super()._task_run(argv, timeout, limits)
|
|
|
|
|
|
def setup(tmp_path, monkeypatch):
|
|
executor = Executor(tmp_path, task_boundary=True)
|
|
store = Store(executor)
|
|
monkeypatch.setattr("openultrasast.search.executor.URLTransport", lambda: store)
|
|
events = []
|
|
lane = SimpleNamespace(
|
|
store=store,
|
|
host="files.example",
|
|
args=SimpleNamespace(atespace="default"),
|
|
workload=batch.Workload("image@sha256:" + "a" * 64, frozenset(), 900, None),
|
|
clock=time.monotonic,
|
|
)
|
|
|
|
def ax(*args, **kwargs):
|
|
events.append(args[0])
|
|
if args[0] == "apply":
|
|
doc = kwargs["manifest"]
|
|
assert doc["metadata"]["name"].startswith("ousast-engine-search-exec-")
|
|
assert {x["name"] for x in doc["spec"]["env"]} == {
|
|
"REPO_URL",
|
|
"COMMAND_GET_URL",
|
|
"EXECUTOR_RESULT_PUT_URL",
|
|
"PREPARED_PUT_URL",
|
|
"PREPARED_PART_URLS",
|
|
}
|
|
|
|
lane.ax = ax
|
|
return executor, lane, events
|
|
|
|
|
|
def test_smoke_mailbox_end_to_end(tmp_path, monkeypatch):
|
|
executor, lane, events = setup(tmp_path, monkeypatch)
|
|
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
|
assert record["status"] == "ok"
|
|
assert executor.stopped
|
|
assert events == ["apply", "delete"]
|
|
assert not lane.store.objects
|
|
assert [r["step"] for r in record["commands"]] == ["clone", "list_files", "read_file", "grep", "install", "trivial", "stop"]
|
|
assert record["scratch_peak_bytes"] >= 8192
|
|
read = next(r for r in record["commands"] if r["step"] == "read_file")
|
|
assert read["input_bytes"] == 10
|
|
assert all(r["wall_seconds"] >= 0 and r["output_bytes"] >= 0 for r in record["commands"])
|
|
|
|
|
|
@pytest.mark.parametrize("kind,reason", [("empty", "clone yielded zero files"), ("empty_install", "empty fast install")])
|
|
def test_instrument_failure_and_cleanup(tmp_path, monkeypatch, kind, reason):
|
|
executor, lane, events = setup(tmp_path, monkeypatch)
|
|
setattr(executor, kind, True)
|
|
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
|
assert record["status"] == "instrument_failure"
|
|
assert reason in record["reason"]
|
|
assert events[-1] == "delete"
|
|
assert not lane.store.objects
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"files,recipe", [(["package.json"], "npm"), (["composer.json"], "composer"), (["pom.xml"], "maven"), (["pyproject.toml"], "pip")]
|
|
)
|
|
def test_recipes(files, recipe):
|
|
assert smoke.install_recipe(files)[0] == recipe
|
|
|
|
|
|
def test_real_executor_measurements(tmp_path):
|
|
(tmp_path / "input").write_bytes(b"12345")
|
|
result = InProcessExecutor(tmp_path, task_boundary=True).submit("run", {"command": ["python3", "-c", "print(1)"]})
|
|
assert result["exit_code"] == 0
|
|
assert result["stdout_bytes"] == 2
|
|
assert result["scratch_peak_bytes"] >= 5
|
|
assert result["wall_seconds"] > 0
|
|
|
|
|
|
def test_docker_managed_task_uses_docker_and_no_heap_flag(tmp_path, monkeypatch):
|
|
import subprocess
|
|
|
|
executor = Executor(tmp_path, task_boundary=True)
|
|
store = Store(executor)
|
|
monkeypatch.setattr("openultrasast.search.executor.URLTransport", lambda: store)
|
|
monkeypatch.setenv("S3_ENDPOINT", "https://files.example")
|
|
calls = []
|
|
|
|
def runner(argv, **kwargs):
|
|
calls.append(argv)
|
|
return subprocess.CompletedProcess(argv, 0, "", "")
|
|
|
|
lane = batch.DockerLane(
|
|
SimpleNamespace(atespace="default"), batch.Workload("image@sha256:" + "a" * 64, frozenset(), 900, None), store=store, runner=runner
|
|
)
|
|
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
|
assert record["status"] == "ok"
|
|
assert [a[1] for a in calls] == ["image", "run", "rm"]
|
|
assert "--heap-profile" not in calls[1]
|
|
assert calls[2][2] == "-f"
|
|
|
|
|
|
def test_apply_exception_still_deletes(tmp_path, monkeypatch):
|
|
executor, lane, events = setup(tmp_path, monkeypatch)
|
|
|
|
def failing(*args, **kwargs):
|
|
events.append(args[0])
|
|
if args[0] == "apply":
|
|
raise OSError("do not expose https://secret.example/token")
|
|
|
|
lane.ax = failing
|
|
result = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
|
assert result["status"] == "instrument_failure"
|
|
assert events == ["apply", "delete"]
|
|
assert "secret.example" not in json.dumps(result)
|
|
|
|
|
|
def test_smoke_retains_sanitized_command_failure(tmp_path, monkeypatch):
|
|
executor, lane, events = setup(tmp_path, monkeypatch)
|
|
original = executor._task_run
|
|
|
|
def fail(argv, timeout, limits):
|
|
if "pip" in argv:
|
|
return {
|
|
"exit_code": 17,
|
|
"stderr": "project failed: Bearer hidden https://host/project?q=secret",
|
|
"stdout": "",
|
|
"phase": "run",
|
|
"reason": "CommandFailure: exit code 17",
|
|
}
|
|
return original(argv, timeout, limits)
|
|
|
|
monkeypatch.setattr(executor, "_task_run", fail)
|
|
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
|
row = record["commands"][-1]
|
|
assert row["exit_code"] == 17 and row["phase"] == "run"
|
|
assert "CommandFailure" in row["reason"]
|
|
assert all(word not in json.dumps(record) for word in ("hidden", "secret", "project", "example"))
|
|
assert events == ["apply", "delete"]
|
|
|
|
|
|
def test_docker_missing_image_is_diagnosable_without_pull(tmp_path, monkeypatch):
|
|
import subprocess
|
|
|
|
monkeypatch.setenv("S3_ENDPOINT", "https://files.example")
|
|
calls = []
|
|
|
|
def runner(argv, **kwargs):
|
|
calls.append(argv)
|
|
return subprocess.CompletedProcess(argv, 1, "", "No such image: pinned-image")
|
|
|
|
lane = batch.DockerLane(
|
|
SimpleNamespace(atespace="default"),
|
|
batch.Workload("image@sha256:" + "a" * 64, frozenset(), 900, None),
|
|
store=Store(Executor(tmp_path, task_boundary=True)),
|
|
runner=runner,
|
|
)
|
|
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
|
assert record["status"] == "instrument_failure"
|
|
assert "ValueError:" in record["reason"] and "No such image" in record["reason"]
|
|
assert [c[1] for c in calls] == ["image"]
|
|
|
|
|
|
@pytest.mark.parametrize("delete_fails", [False, True])
|
|
def test_capacity_partial_create_deleted_before_retry(tmp_path, monkeypatch, delete_fails):
|
|
executor, lane, events = setup(tmp_path, monkeypatch)
|
|
live, attempts = set(), []
|
|
|
|
def ax(*args, **kwargs):
|
|
events.append(args[0])
|
|
if args[0] == "apply":
|
|
assert not live
|
|
live.add(kwargs["manifest"]["metadata"]["name"])
|
|
attempts.append(1)
|
|
if len(attempts) == 1:
|
|
raise batch.NoFreeWorkers("partially provisioned")
|
|
elif args[0] == "delete":
|
|
if delete_fails:
|
|
raise OSError("cannot delete")
|
|
live.discard(args[2])
|
|
|
|
lane.ax, lane.pause = ax, lambda seconds: None
|
|
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
|
assert len(attempts) == (1 if delete_fails else 2)
|
|
assert record["status"] == ("instrument_failure" if delete_fails else "ok")
|
|
if not delete_fails:
|
|
assert events == ["apply", "delete", "apply", "delete"] and not live
|
|
else:
|
|
# Simulated leaked task deliberately retains its reservation; restore test global.
|
|
batch.AX_SLOTS.release()
|
|
|
|
|
|
def test_apply_timeout_does_not_resubmit(tmp_path, monkeypatch):
|
|
import subprocess
|
|
|
|
executor, lane, events = setup(tmp_path, monkeypatch)
|
|
|
|
def ax(*args, **kwargs):
|
|
events.append(args[0])
|
|
if args[0] == "apply":
|
|
raise subprocess.TimeoutExpired("apply", 30)
|
|
|
|
lane.ax = ax
|
|
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
|
assert events == ["apply", "delete"]
|
|
assert "TimeoutExpired:" in record["reason"]
|
|
|
|
|
|
def test_dispatch_rejects_duplicate_items_before_launch(tmp_path):
|
|
item = batch.Item("same", {}, "result.json", (), {}, "")
|
|
|
|
def unexpected(*args):
|
|
pytest.fail("duplicate inputs must be rejected before any dispatch")
|
|
|
|
with pytest.raises(ValueError, match="duplicate dispatch item"):
|
|
batch.dispatch([item, item], batch.Workload("image", frozenset(), 1, None), tmp_path, {"ax": unexpected}, lanes=("ax",))
|
|
|
|
|
|
@pytest.mark.parametrize("stage", ["enter", "body", "normal"])
|
|
def test_cleanup_preserves_original_and_logs(tmp_path, monkeypatch, caplog, stage):
|
|
_, lane, _ = setup(tmp_path, monkeypatch)
|
|
original = ValueError("original apply failure")
|
|
|
|
def ax(*args, **kwargs):
|
|
if args[0] == "apply" and stage == "enter":
|
|
raise original
|
|
if args[0] == "delete":
|
|
raise ValueError("delete failed https://private.example/?token=secret")
|
|
|
|
lane.ax = ax
|
|
# Retained reservations must not leak into other tests.
|
|
monkeypatch.setattr(batch, "AX_SLOTS", __import__("threading").BoundedSemaphore(2))
|
|
with pytest.raises(RuntimeError if stage == "normal" else ValueError) as caught, batch.SearchExecutorTask(lane, b"archive"):
|
|
if stage == "body":
|
|
raise original
|
|
if stage != "normal":
|
|
assert caught.value is original
|
|
assert str(caught.value).startswith("original apply failure; executor cleanup failed:")
|
|
assert "delete failed" in str(caught.value)
|
|
assert "executor cleanup failed" in caplog.text
|
|
assert "private.example" not in caplog.text + str(caught.value)
|