marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

search-with-proof: diagnosable failures, pip install race, no duplicate tasks

Browse Source 
Reasons carry type plus a sanitised message tail; command failures carry phase,
exit code and bounded stderr. Scratch walk tolerates directories pip removes
mid-walk; writable /workspace/home and runtime paths. Capacity retries delete a
possibly created task before resubmitting. Docker smoke checks the image first.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-05 17:58:27 +02:00
co-authored by Claude Opus 5.5
parent 1d67412e2b
commit 941fbb5fb0
15 changed files with 446 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/ax/batch.py
+32 -22
View File
@@ -25,6 +25,7 @@ import yaml
from openultrasast.config import load_dotenv
from openultrasast.plane.memory import PRESIGNED_ARCHIVE_SUFFIX, S3Store, open_store
from openultrasast.search.task_storage import diagnostic, exception_reason
class EgressError(ValueError):
@@ -54,6 +55,12 @@ def apply_when_available(lane, document, end):
lane.ax("apply", "-f", "-", manifest=document, timeout=min(30, end - lane.clock()))
return
except NoFreeWorkers:
# A failed apply is not proof that the server did not create a task.
# Never send another create while the preceding task can still run.
try:
lane.ax("delete", "task", document["metadata"]["name"])
except Exception as exc:
raise RuntimeError("capacity cleanup failed: " + exception_reason(exc)) from None
lane.pause(min(random.uniform(15, 30), max(0, end - lane.clock())))
raise CapacityDeadline("no free workers available before deadline")
@@ -185,7 +192,11 @@ class AXLane:
if slot:
AX_SLOTS.release()
if done is None or done.returncode:
return {"status": "instrument_failure", "ax_attempts": attempts}
return {
"status": "instrument_failure",
"reason": diagnostic(done.stderr) if done is not None else "TimeoutError: task result deadline",
"ax_attempts": attempts,
}
record = json.loads((output / "result.json").read_text())
return {**record, "worker_ip": worker, "ax_attempts": attempts}
@@ -203,9 +214,11 @@ class AXLane:
)
if done.returncode:
if args[0] == "apply" and "no free workers available" in (done.stdout + done.stderr).lower():
raise NoFreeWorkers
# AX errors may echo manifest URLs; report only verb and exit code.
raise ValueError(f"ax {args[0]} failed (exit {done.returncode})")
raise NoFreeWorkers("no free workers available")
if args[0] == "delete" and "notfound" in (done.stdout + done.stderr).lower().replace(" ", ""):
return done
# AX errors may echo manifest URLs; sanitize before recording.
raise ValueError(f"ax {args[0]} failed (exit {done.returncode}): " + diagnostic(done.stderr))
return done
def _execute_attempt(self, item, output, attempt, attempts):
@@ -266,13 +279,13 @@ class AXLane:
raise SandboxFailure
self.pause(min(2, max(0, end - self.clock())))
return None, worker_ip
except (subprocess.TimeoutExpired, TimeoutError):
return None, worker_ip
except (subprocess.TimeoutExpired, TimeoutError) as exc:
return subprocess.CompletedProcess([], 1, "", exception_reason(exc)), worker_ip
except (SandboxFailure, CapacityDeadline):
raise
except Exception as exc:
# Never expose exception strings which may contain object-scoped bearer URLs.
detail = str(exc) if isinstance(exc, EgressError) else type(exc).__name__
detail = exception_reason(exc)
return subprocess.CompletedProcess([], 1, "", "AX transport failed: " + detail), worker_ip
finally:
attempts.append({"name": name, "phase": phase, "seconds": self.clock() - started})
@@ -283,12 +296,12 @@ class AXLane:
except Exception as exc:
# A successful list already confirmed a disappeared task is gone.
if phase != "Missing":
errors.append("task: " + type(exc).__name__)
errors.append("task: " + exception_reason(exc))
for key in keys:
try:
self.store._delete(key)
except Exception as exc:
errors.append("object: " + type(exc).__name__)
errors.append("object: " + exception_reason(exc))
if errors:
raise RuntimeError("AX cleanup failed; stop dispatching: " + ", ".join(errors))
@@ -414,6 +427,9 @@ def dispatch(items, workload, out, executors, *, lanes=("ax", "ax", "docker"), o
if not isinstance(max_attempts, int) or not 1 <= max_attempts <= 2:
raise ValueError("batch attempts must be one or two")
out.mkdir(parents=True, exist_ok=True)
items = list(items)
if len({item.id for item in items}) != len(items):
raise ValueError("duplicate dispatch item id")
pending, fallbacks = [], []
def path(item):
@@ -465,15 +481,7 @@ def dispatch(items, workload, out, executors, *, lanes=("ax", "ax", "docker"), o
def docker_diagnostic(text):
"""Bound diagnostics and remove bearer URLs, environment secrets and assignments."""
text = str(text or "")
for key, value in os.environ.items():
if len(value) >= 4 and any(word in key.upper() for word in ("TOKEN", "SECRET", "PASSWORD", "CREDENTIAL", "KEY")):
text = text.replace(value, "[redacted]")
text = re.sub(r"https?://[^\s\"']+", "[redacted-url]", text)
text = re.sub(
r"(?i)(authorization|bearer|password|token|secret|signature|credential|[\w-]*key)\s*[:= ]\s*[^\s,;]+", r"\1=[redacted]", text
)
return "\n".join(text.splitlines()[-10:])[-4000:]
return diagnostic(text or "", maximum=4000)
class DockerLane(AXLane):
@@ -510,12 +518,12 @@ class DockerLane(AXLane):
done = self.runner(command, capture_output=True, text=True, timeout=timeout)
except (OSError, subprocess.SubprocessError) as exc:
self._diagnostics.append(
{"step": command[1], "exit_code": None, "stderr": docker_diagnostic(getattr(exc, "stderr", "") or type(exc).__name__)}
{"step": command[1], "exit_code": None, "stderr": docker_diagnostic(getattr(exc, "stderr", "") or exception_reason(exc))}
)
raise
if done.returncode:
self._diagnostics.append({"step": command[1], "exit_code": done.returncode, "stderr": docker_diagnostic(done.stderr)})
raise ValueError(f"docker {command[1]} failed (exit {done.returncode})")
raise ValueError(f"docker {command[1]} failed (exit {done.returncode}): " + diagnostic(done.stderr))
return done
def _prepare_image(self):
@@ -625,6 +633,8 @@ class SearchExecutorTask:
expires = timedelta(seconds=self.deadline + 600)
store = self.lane.store
try:
if isinstance(self.lane, DockerLane):
self.lane._run_docker(["docker", "image", "inspect", self.lane.workload.image], min(30, self.deadline))
store._put(self.keys[0], self.archive)
urls = {
"REPO_URL": store.presign_get(self.keys[0], expires),
@@ -684,12 +694,12 @@ class SearchExecutorTask:
try:
self.lane.ax("delete", "task", self.name)
except Exception as error:
errors.append(type(error).__name__)
errors.append(exception_reason(error))
for key in self.keys:
try:
self.lane.store._delete(key)
except Exception as error:
errors.append(type(error).__name__)
errors.append(exception_reason(error))
if self.slot and not errors:
AX_SLOTS.release()
self.slot = False
benchmarks/search/executor_smoke.py
+35 -29
View File
@@ -1,7 +1,7 @@
"""One no-model executor task; run as python -m benchmarks.search.executor_smoke.
Runtime acquisition/install needs network inside the executor. Unit tests are offline.
Records contain measurements, never command output or presigned bearer URLs.
Records contain measurements and sanitized failure tails, never presigned bearer URLs.
"""
from __future__ import annotations
@@ -16,6 +16,7 @@ from pathlib import Path
from urllib.parse import urlsplit
from benchmarks.ax.batch import AXLane, DockerLane, SearchExecutorTask, Workload, validate_image
from openultrasast.search.task_storage import diagnostic, exception_reason
# URL and pin are positional shell arguments, never interpolated into shell code.
# Retry the initial acquisition for delayed task egress activation (at most 60 s).
@@ -55,6 +56,8 @@ def run_smoke(lane, repo_url, commit, *, deadline=900):
"scratch_measurement": "sampled workspace allocated bytes plus command logs",
"model_calls": 0,
}
parts = urlsplit(repo_url)
private = tuple(p for p in parts.path.split("/") if p) + (parts.path.rsplit("/", 1)[-1].removesuffix(".git"),)
started = time.monotonic()
try:
parts = urlsplit(repo_url)
@@ -77,9 +80,18 @@ def run_smoke(lane, repo_url, commit, *, deadline=900):
if name == "run":
args = {**args, "timeout_seconds": timeout}
before = time.monotonic()
result = task.client.submit(
name, args, timeout_seconds=timeout, limits={"memory_bytes": 2 * 1024**3, "disk_bytes": 2 * 1024**3}
)
try:
result = task.client.submit(
name, args, timeout_seconds=timeout, limits={"memory_bytes": 2 * 1024**3, "disk_bytes": 2 * 1024**3}
)
except Exception as exc:
result = {
"error": type(exc).__name__,
"reason": exception_reason(exc, private=private),
"phase": "timeout" if isinstance(exc, TimeoutError) else "result upload",
"exit_code": None,
"stderr": "",
}
output_bytes = sum(result.get(k + "_bytes", len(result.get(k, "").encode())) for k in ("stdout", "stderr"))
if name != "run":
output_bytes = len(json.dumps(result).encode())
@@ -95,12 +107,27 @@ def run_smoke(lane, repo_url, commit, *, deadline=900):
}
if "input_bytes" in result:
row["input_bytes"] = result["input_bytes"]
failed = (
result.get("error")
or result.get("timed_out")
or result.get("status") == "could_not_build"
or (name == "run" and result.get("exit_code") != 0)
)
if failed:
row.update(
phase=result.get("phase", "timeout" if result.get("timed_out") else "run"),
reason=diagnostic(
result.get("reason") or result.get("error") or f"CommandFailure: exit code {result.get('exit_code')}",
private=private,
),
stderr=diagnostic(result.get("stderr", ""), maximum=4000, private=private),
)
record["commands"].append(row)
record["scratch_peak_bytes"] = max(record["scratch_peak_bytes"], result.get("scratch_peak_bytes", 0))
if result.get("error") or result.get("timed_out") or result.get("status") == "could_not_build":
raise ValueError(step + " executor failure")
raise ValueError(step + " executor failure: " + row["reason"])
if name == "run" and result.get("exit_code") != 0:
raise ValueError(step + " nonzero or missing exit code")
raise ValueError(step + " nonzero or missing exit code: " + row["reason"])
if step == "install" and output_bytes == 0 and row["wall_seconds"] < 1:
raise ValueError("empty fast install: exit 0 with empty output in under 1 s")
return result
@@ -130,28 +157,7 @@ def run_smoke(lane, repo_url, commit, *, deadline=900):
record["status"] = "ok"
except Exception as exc:
record["status"] = "instrument_failure"
# Only our fixed diagnostic strings are safe; transport errors may contain URLs.
record["reason"] = (
str(exc)
if isinstance(exc, ValueError)
and str(exc)
in {
"credential-free HTTPS repository URL required",
"full lowercase commit SHA required",
"no recognised install manifest",
"clone yielded zero files",
"file read lacks input size",
"trivial command did not print 1",
"stop not acknowledged",
"empty fast install: exit 0 with empty output in under 1 s",
*(
step + suffix
for step in ("clone", "list_files", "read_file", "grep", "install", "trivial", "stop")
for suffix in (" executor failure", " nonzero or missing exit code")
),
}
else type(exc).__name__
)
record["reason"] = exception_reason(exc, private=private)
record["wall_seconds"] = time.monotonic() - started
return record
@@ -173,7 +179,7 @@ def main(argv=None):
lane = (DockerLane if args.lane == "docker" else AXLane)(args, Workload(args.image, frozenset(), args.deadline, None))
record = run_smoke(lane, args.repo_url, args.commit, deadline=args.deadline)
except Exception as exc:
record = {"status": "instrument_failure", "reason": "lane setup: " + type(exc).__name__}
record = {"status": "instrument_failure", "reason": "lane setup: " + exception_reason(exc)}
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n")
print(json.dumps(record))
benchmarks/search/pilot_select.py
+11 -1
View File
@@ -19,6 +19,7 @@ from urllib.parse import urlsplit
from benchmarks.unseen import eligibility, ledger
from openultrasast.model.taxonomy import load_families
from openultrasast.search.task_storage import exception_reason
LANGUAGES = ("python", "javascript", "typescript", "php", "java")
FAMILIES = ("injection", "path", "output_encoding", "untrusted_destination")
@@ -352,7 +353,16 @@ def main(argv=None) -> int:
except Exception as exc:
if created:
private.unlink(missing_ok=True)
print(json.dumps({"status": "instrument_failure", "reason": type(exc).__name__}))
print(
json.dumps(
{
"status": "instrument_failure",
"reason": exception_reason(
exc, private=tuple(part for row in locals().get("rows", []) for part in str(row.get("repo", "")).split("/") if part)
),
}
)
)
return 1