mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
search-with-proof: executor smoke driver and pilot selection (6.1)
20 development pairs, one per repository, 4 per language, 5 per family; identities in a gitignored manifest, counts and digest committed. Executor smoke drives one executor task end to end without a model and fails loudly on an unread clone. kube-ax operator constraints as steering. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
85f4f0b3d4
commit
1d67412e2b
@@ -0,0 +1,162 @@
|
||||
"""No-network end-to-end executor smoke, through the real mailbox client."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from benchmarks.ax import batch
|
||||
from benchmarks.search import executor_smoke as smoke
|
||||
from openultrasast.search.executor import Command, InProcessExecutor
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, executor):
|
||||
self.executor, self.objects = executor, {}
|
||||
|
||||
def _put(self, key, data):
|
||||
self.objects[key] = data
|
||||
|
||||
def _delete(self, key):
|
||||
self.objects.pop(key, None)
|
||||
|
||||
def presign_get(self, key, expires):
|
||||
return "https://files.example/" + key
|
||||
|
||||
presign_put = presign_get
|
||||
|
||||
def put(self, url, data, timeout):
|
||||
self.result = json.dumps(self.executor.execute(Command(**json.loads(data)))).encode()
|
||||
|
||||
def get(self, url, timeout):
|
||||
return self.result
|
||||
|
||||
|
||||
class Executor(InProcessExecutor):
|
||||
empty = False
|
||||
empty_install = False
|
||||
|
||||
def _task_run(self, argv, timeout, limits):
|
||||
if argv[:2] == ["sh", "-ec"]:
|
||||
if not self.empty:
|
||||
(self.repo / "README.md").write_text("test input")
|
||||
(self.repo / "requirements.txt").write_text("")
|
||||
return dict(exit_code=0, stdout="cloned", stderr="", scratch_peak_bytes=4096, stdout_bytes=6, stderr_bytes=0)
|
||||
if "pip" in argv:
|
||||
return dict(
|
||||
exit_code=0,
|
||||
stdout="" if self.empty_install else "installed",
|
||||
stderr="",
|
||||
scratch_peak_bytes=8192,
|
||||
stdout_bytes=0 if self.empty_install else 9,
|
||||
stderr_bytes=0,
|
||||
)
|
||||
return super()._task_run(argv, timeout, limits)
|
||||
|
||||
|
||||
def setup(tmp_path, monkeypatch):
|
||||
executor = Executor(tmp_path, task_boundary=True)
|
||||
store = Store(executor)
|
||||
monkeypatch.setattr("openultrasast.search.executor.URLTransport", lambda: store)
|
||||
events = []
|
||||
lane = SimpleNamespace(
|
||||
store=store,
|
||||
host="files.example",
|
||||
args=SimpleNamespace(atespace="default"),
|
||||
workload=batch.Workload("image@sha256:" + "a" * 64, frozenset(), 900, None),
|
||||
clock=time.monotonic,
|
||||
)
|
||||
|
||||
def ax(*args, **kwargs):
|
||||
events.append(args[0])
|
||||
if args[0] == "apply":
|
||||
doc = kwargs["manifest"]
|
||||
assert doc["metadata"]["name"].startswith("ousast-engine-search-exec-")
|
||||
assert {x["name"] for x in doc["spec"]["env"]} == {"REPO_URL", "COMMAND_GET_URL", "EXECUTOR_RESULT_PUT_URL", "PREPARED_PUT_URL"}
|
||||
|
||||
lane.ax = ax
|
||||
return executor, lane, events
|
||||
|
||||
|
||||
def test_smoke_mailbox_end_to_end(tmp_path, monkeypatch):
|
||||
executor, lane, events = setup(tmp_path, monkeypatch)
|
||||
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
||||
assert record["status"] == "ok"
|
||||
assert executor.stopped
|
||||
assert events == ["apply", "delete"]
|
||||
assert not lane.store.objects
|
||||
assert [r["step"] for r in record["commands"]] == ["clone", "list_files", "read_file", "grep", "install", "trivial", "stop"]
|
||||
assert record["scratch_peak_bytes"] >= 8192
|
||||
read = next(r for r in record["commands"] if r["step"] == "read_file")
|
||||
assert read["input_bytes"] == 10
|
||||
assert all(r["wall_seconds"] >= 0 and r["output_bytes"] >= 0 for r in record["commands"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind,reason", [("empty", "clone yielded zero files"), ("empty_install", "empty fast install")])
|
||||
def test_instrument_failure_and_cleanup(tmp_path, monkeypatch, kind, reason):
|
||||
executor, lane, events = setup(tmp_path, monkeypatch)
|
||||
setattr(executor, kind, True)
|
||||
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
||||
assert record["status"] == "instrument_failure"
|
||||
assert reason in record["reason"]
|
||||
assert events[-1] == "delete"
|
||||
assert not lane.store.objects
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"files,recipe", [(["package.json"], "npm"), (["composer.json"], "composer"), (["pom.xml"], "maven"), (["pyproject.toml"], "pip")]
|
||||
)
|
||||
def test_recipes(files, recipe):
|
||||
assert smoke.install_recipe(files)[0] == recipe
|
||||
|
||||
|
||||
def test_real_executor_measurements(tmp_path):
|
||||
(tmp_path / "input").write_bytes(b"12345")
|
||||
result = InProcessExecutor(tmp_path, task_boundary=True).submit("run", {"command": ["python3", "-c", "print(1)"]})
|
||||
assert result["exit_code"] == 0
|
||||
assert result["stdout_bytes"] == 2
|
||||
assert result["scratch_peak_bytes"] >= 5
|
||||
assert result["wall_seconds"] > 0
|
||||
|
||||
|
||||
def test_docker_managed_task_uses_docker_and_no_heap_flag(tmp_path, monkeypatch):
|
||||
import subprocess
|
||||
|
||||
executor = Executor(tmp_path, task_boundary=True)
|
||||
store = Store(executor)
|
||||
monkeypatch.setattr("openultrasast.search.executor.URLTransport", lambda: store)
|
||||
monkeypatch.setenv("S3_ENDPOINT", "https://files.example")
|
||||
calls = []
|
||||
|
||||
def runner(argv, **kwargs):
|
||||
calls.append(argv)
|
||||
return subprocess.CompletedProcess(argv, 0, "", "")
|
||||
|
||||
lane = batch.DockerLane(
|
||||
SimpleNamespace(atespace="default"), batch.Workload("image@sha256:" + "a" * 64, frozenset(), 900, None), store=store, runner=runner
|
||||
)
|
||||
record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
||||
assert record["status"] == "ok"
|
||||
assert [a[1] for a in calls] == ["run", "rm"]
|
||||
assert "--heap-profile" not in calls[0]
|
||||
assert calls[1][2] == "-f"
|
||||
|
||||
|
||||
def test_apply_exception_still_deletes(tmp_path, monkeypatch):
|
||||
executor, lane, events = setup(tmp_path, monkeypatch)
|
||||
|
||||
def failing(*args, **kwargs):
|
||||
events.append(args[0])
|
||||
if args[0] == "apply":
|
||||
raise OSError("do not expose https://secret.example/token")
|
||||
|
||||
lane.ax = failing
|
||||
result = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
|
||||
assert result["status"] == "instrument_failure"
|
||||
assert events == ["apply", "delete"]
|
||||
assert "secret.example" not in json.dumps(result)
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Offline pilot selection: synthetic repositories only."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
patch.syspath_prepend(str(Path(__file__).resolve().parents[1]))
|
||||
from benchmarks.search import pilot_select as pilot
|
||||
|
||||
|
||||
def pair(i, language="python", family="injection", **extra):
|
||||
return {
|
||||
"repo": f"https://github.com/synthetic/project-{i}",
|
||||
"parent": "a" * 40,
|
||||
"commit": "b" * 40,
|
||||
"language": language,
|
||||
"family": family,
|
||||
"split": "train",
|
||||
"path": "src/main.py",
|
||||
"expected": [{"function": "handler"}],
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def test_balanced_unique_deterministic_selection():
|
||||
rows = [pair(f"{lang}-{i}", lang, pilot.FAMILIES[i % 4]) for lang in pilot.LANGUAGES for i in range(8)]
|
||||
candidates, rejected = pilot.candidates(rows, set())
|
||||
selected = pilot.select(candidates, seed=17)
|
||||
assert len(selected) == len({r["repo"] for r in selected}) == 20
|
||||
assert all(sum(r["language"] == lang for r in selected) >= 3 for lang in pilot.LANGUAGES)
|
||||
assert set(r["family"] for r in selected) == set(pilot.FAMILIES)
|
||||
assert selected == pilot.select(list(reversed(candidates)), seed=17)
|
||||
assert selected != pilot.select(candidates, seed=18)
|
||||
assert not rejected
|
||||
|
||||
|
||||
def test_filter_non_development_bad_commits_families_and_used():
|
||||
rows = [
|
||||
pair(1),
|
||||
pair(2, split="holdout"),
|
||||
pair(3, family="access_control"),
|
||||
pair(4, parent="b" * 40),
|
||||
pair(5),
|
||||
pair(6, parent="bad"),
|
||||
pair(7, repo="http://github.com/synthetic/no"),
|
||||
]
|
||||
selected, rejected = pilot.candidates(rows, {"synthetic/project-5"})
|
||||
assert len(selected) == 1
|
||||
assert sum(rejected.values()) == 6
|
||||
|
||||
|
||||
def test_prefer_evidenced_test_suite_and_manifest_and_unique_repos():
|
||||
rows = [pair(1), pair(2, files=["pyproject.toml", "tests/test_main.py"]), pair(2, family="path")]
|
||||
selected = pilot.select(pilot.candidates(rows, set())[0], seed=2, count=2)
|
||||
assert selected[0]["repo"].endswith("project-2")
|
||||
assert len({row["repo"] for row in selected}) == 2
|
||||
assert selected[0]["has_tests"] and selected[0]["manifest"] == "pip"
|
||||
|
||||
|
||||
def test_unseen_local_pool_used_and_ledger(tmp_path):
|
||||
unseen = tmp_path / "benchmarks/unseen"
|
||||
unseen.mkdir(parents=True)
|
||||
(unseen / "pool-p1.toml").write_text('[[repository]]\nrepo="https://github.com/synthetic/project-1"\n')
|
||||
(unseen / "used.json").write_text(json.dumps({"repo": "synthetic/project-2"}))
|
||||
names, report = pilot.exclusions(tmp_path)
|
||||
assert names == {"synthetic/project-1", "synthetic/project-2"}
|
||||
assert report["files"] == 2 and report["bytes"] > 0
|
||||
(unseen / "usage.jsonl").write_text('{"bad":true}\n')
|
||||
with pytest.raises(ValueError, match="broken_digest_chain"):
|
||||
pilot.exclusions(tmp_path)
|
||||
|
||||
|
||||
def test_never_open_reserved_or_symlink_catalogs(tmp_path):
|
||||
root = tmp_path / "benchmarks/pairs"
|
||||
root.mkdir(parents=True)
|
||||
(root / "catalog.toml").write_text('[[pair]]\nrepo="synthetic/one"\n')
|
||||
hidden = root / "population-v3"
|
||||
hidden.mkdir()
|
||||
(hidden / "catalog.toml").write_bytes(b"not toml")
|
||||
(root / "alias").symlink_to(hidden, target_is_directory=True)
|
||||
rows, counts = pilot.load_catalogs(tmp_path)
|
||||
assert len(rows) == 1 and counts["files"] == 1
|
||||
assert counts["bytes"] > 0
|
||||
|
||||
|
||||
def test_summary_contains_counts_and_digests_not_names():
|
||||
selected = pilot.select(pilot.candidates([pair(1)], set())[0], seed=1)
|
||||
summary = pilot.summary(selected, seed=1, manifest=b"private")
|
||||
assert summary["selected"] == 1 and summary["gap_to_target"] == 19
|
||||
assert "synthetic" not in json.dumps(summary)
|
||||
assert summary["unknown_test_suite"] == 1
|
||||
|
||||
|
||||
def test_bare_used_names_and_repository_alias_fields(tmp_path):
|
||||
unseen = tmp_path / "benchmarks/unseen"
|
||||
unseen.mkdir(parents=True)
|
||||
(unseen / "used.json").write_text(json.dumps({"used": ["synthetic/project-1"]}))
|
||||
assert pilot.exclusions(tmp_path)[0] == {"synthetic/project-1"}
|
||||
|
||||
|
||||
def test_cross_repository_and_noncanonical_urls_rejected():
|
||||
rows = [
|
||||
pair(1, fix_repo="synthetic/other"),
|
||||
pair(2, repo="https://user:[email protected]/synthetic/project-2"),
|
||||
pair(3, repo="https://github.com/synthetic/project-3?query=1"),
|
||||
pair(4, repo="https://github.com/synthetic/project-4/extra"),
|
||||
]
|
||||
assert not pilot.candidates(rows, set())[0]
|
||||
|
||||
|
||||
def test_metadata_from_matching_local_checkout_only(tmp_path):
|
||||
checkout = tmp_path / "project-1"
|
||||
(checkout / ".git").mkdir(parents=True)
|
||||
(checkout / ".git/config").write_text('[remote "origin"]\nurl=https://github.com/synthetic/project-1\n')
|
||||
(checkout / "pyproject.toml").write_text('[project]\nname="synthetic"\n')
|
||||
(checkout / "tests").mkdir()
|
||||
(checkout / "tests/test_main.py").write_text("def test_smoke(): assert True\n")
|
||||
rows = [pair(1), pair(2)]
|
||||
counts = pilot.enrich(rows, (tmp_path,))
|
||||
assert counts["matched_checkouts"] == 1
|
||||
selected = pilot.candidates(rows, set())[0]
|
||||
assert selected[0]["manifest"] == "pip" and selected[0]["has_tests"] is True
|
||||
assert selected[1]["manifest"] is None
|
||||
|
||||
|
||||
def test_cache_enrichment_never_follows_reserved_paths(tmp_path, monkeypatch):
|
||||
cache = tmp_path / "cache"
|
||||
checkout = cache / "project-1"
|
||||
(checkout / ".git").mkdir(parents=True)
|
||||
reserved = tmp_path / "population-v3"
|
||||
reserved.mkdir()
|
||||
config = reserved / "config"
|
||||
config.write_text('[remote "origin"]\nurl=https://github.com/synthetic/project-1\n')
|
||||
(checkout / ".git/config").symlink_to(config)
|
||||
original = Path.read_bytes
|
||||
|
||||
def guarded(path):
|
||||
if "population-v3" in path.resolve().parts:
|
||||
pytest.fail("reserved file opened")
|
||||
return original(path)
|
||||
|
||||
monkeypatch.setattr(Path, "read_bytes", guarded)
|
||||
assert pilot.enrich([pair(1)], (cache,))["matched_checkouts"] == 0
|
||||
(checkout / ".git/config").unlink()
|
||||
(checkout / ".git/config").write_text(config.read_text())
|
||||
(checkout / "tests").symlink_to(reserved, target_is_directory=True)
|
||||
(checkout / "pyproject.toml").symlink_to(config)
|
||||
rows = [pair(1)]
|
||||
pilot.enrich(rows, (cache,))
|
||||
assert not rows[0]["files"]
|
||||
|
||||
|
||||
def test_enrichment_skips_unseen_before_opening_cache(tmp_path, monkeypatch):
|
||||
def fail(*args):
|
||||
pytest.fail("unseen cache read")
|
||||
|
||||
monkeypatch.setattr(Path, "read_bytes", fail)
|
||||
assert pilot.enrich([pair(1)], (tmp_path,), excluded={"synthetic/project-1"})["matched_checkouts"] == 0
|
||||
|
||||
|
||||
def test_cache_enrichment_refuses_ancestor_symlinks(tmp_path, monkeypatch):
|
||||
reserved = tmp_path / "population-v3"
|
||||
checkout = reserved / "project-1"
|
||||
(checkout / ".git").mkdir(parents=True)
|
||||
(checkout / ".git/config").write_text('[remote "origin"]\nurl=https://github.com/synthetic/project-1\n')
|
||||
alias = tmp_path / "alias"
|
||||
alias.symlink_to(reserved, target_is_directory=True)
|
||||
|
||||
def fail(*args):
|
||||
pytest.fail("reserved ancestor opened")
|
||||
|
||||
monkeypatch.setattr(Path, "read_bytes", fail)
|
||||
assert pilot.enrich([pair(1)], (alias,))["matched_checkouts"] == 0
|
||||
Reference in New Issue
Block a user