mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 4.8
649bd42fc3
Make the >=90% recall / <10% FP detection gate a first-class, score-independent CI step with non-regression guards. - openultrasast.gate: canonical corpus map + DetectionMetrics/DetectionGateVerdict, evaluate_corpus/detection_gate (no score input — cannot be unblocked by score), merge_gate (ANDs the hard detection gate with the advisory-first score gate), within_tolerance, and a `python -m openultrasast.gate` entrypoint that exits non-zero on a breach. Wired as an explicit step in both the GitHub and Dagger pipelines. - 8.1: pipeline tests prove a high/passing score cannot unblock a recall-breaching change; the gate fails below 90% recall or at/above 10% FP. - 8.2: tests/golden/detection_baseline.json + a byte-identical baseline test for the default ruleset / empty ledger (44/47 = 93.62% recall, 0% FP). - 8.3: within_tolerance band logic + a capability-gated HarnessX tolerance test. - 8.4: zero-dependency guard — governance/scoring/benchmark planes never import HarnessX (sys.modules check + module-scope import scan). - test_detection_benchmarks now sources the corpus/aggregation from the gate module. 128 passed, 3 skipped; ruff + mypy clean; detection gate PASS. Co-Authored-By: Claude Opus 4.8 <[email protected]>
78 lines
2.4 KiB
Python
78 lines
2.4 KiB
Python
from __future__ import annotations
|
|
|
|
import sys
|
|
|
|
import anyio
|
|
|
|
import dagger
|
|
|
|
# Substrings that mark a transient registry / image-pull failure (e.g. a Docker
|
|
# Hub 5xx on the python:3.11-bookworm pull, or a registry.dagger.io timeout) that
|
|
# is worth retrying rather than failing the whole pipeline.
|
|
_TRANSIENT_MARKERS = (
|
|
"pull image",
|
|
"failed to authorize",
|
|
"failed to fetch oauth",
|
|
"TransportQueryError",
|
|
"i/o timeout",
|
|
"connection reset",
|
|
"context deadline exceeded",
|
|
" 502",
|
|
" 503",
|
|
" 520",
|
|
" 429",
|
|
)
|
|
|
|
|
|
async def _run_checks() -> None:
|
|
config = dagger.Config(log_output=sys.stderr)
|
|
async with dagger.Connection(config) as client:
|
|
source = client.host().directory(
|
|
".",
|
|
exclude=[
|
|
".git",
|
|
".mypy_cache",
|
|
".pytest_cache",
|
|
".ruff_cache",
|
|
".venv",
|
|
"**/.openultrasast",
|
|
"dist",
|
|
],
|
|
)
|
|
checks = (
|
|
client.container()
|
|
.from_("python:3.11-bookworm")
|
|
.with_directory("/src", source)
|
|
.with_workdir("/src")
|
|
.with_exec(["python", "-m", "pip", "install", "uv"])
|
|
.with_exec(["uv", "sync", "--group", "dev"])
|
|
.with_exec(["uv", "run", "ruff", "format", "--check", "."])
|
|
.with_exec(["uv", "run", "ruff", "check", "."])
|
|
.with_exec(["uv", "run", "mypy", "src/openultrasast"])
|
|
.with_exec(["uv", "run", "pytest"])
|
|
.with_exec(["uv", "run", "python", "-m", "openultrasast.gate"])
|
|
.with_exec(["uv", "run", "python", "-m", "compileall", "src", "tests"])
|
|
.with_exec(["uv", "build"])
|
|
)
|
|
await checks.sync()
|
|
|
|
|
|
async def main() -> None:
|
|
attempts = 3
|
|
for attempt in range(1, attempts + 1):
|
|
try:
|
|
await _run_checks()
|
|
return
|
|
except Exception as exc: # noqa: BLE001 — retry only transient registry/pull failures
|
|
message = str(exc)
|
|
transient = any(marker in message for marker in _TRANSIENT_MARKERS)
|
|
if attempt < attempts and transient:
|
|
print(f"dagger: transient registry/pull failure on attempt {attempt}; retrying in 20s", file=sys.stderr)
|
|
await anyio.sleep(20)
|
|
continue
|
|
raise
|
|
|
|
|
|
if __name__ == "__main__":
|
|
anyio.run(main)
|