mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 4.8
649bd42fc3
Make the >=90% recall / <10% FP detection gate a first-class, score-independent CI step with non-regression guards. - openultrasast.gate: canonical corpus map + DetectionMetrics/DetectionGateVerdict, evaluate_corpus/detection_gate (no score input — cannot be unblocked by score), merge_gate (ANDs the hard detection gate with the advisory-first score gate), within_tolerance, and a `python -m openultrasast.gate` entrypoint that exits non-zero on a breach. Wired as an explicit step in both the GitHub and Dagger pipelines. - 8.1: pipeline tests prove a high/passing score cannot unblock a recall-breaching change; the gate fails below 90% recall or at/above 10% FP. - 8.2: tests/golden/detection_baseline.json + a byte-identical baseline test for the default ruleset / empty ledger (44/47 = 93.62% recall, 0% FP). - 8.3: within_tolerance band logic + a capability-gated HarnessX tolerance test. - 8.4: zero-dependency guard — governance/scoring/benchmark planes never import HarnessX (sys.modules check + module-scope import scan). - test_detection_benchmarks now sources the corpus/aggregation from the gate module. 128 passed, 3 skipped; ruff + mypy clean; detection gate PASS. Co-Authored-By: Claude Opus 4.8 <[email protected]>
36 lines
1.3 KiB
Python
36 lines
1.3 KiB
Python
"""Detection-quality gate for the in-scope languages.
|
|
|
|
Runs quick-mode scans against the C/C++, Python, JavaScript, and Java benchmark
|
|
corpora and asserts the project goal: at least 90% recall of known vulnerabilities
|
|
and under a 10% false-positive rate per language. The corpus map and aggregation
|
|
live in ``openultrasast.gate`` (the same gate CI runs as a first-class step).
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from openultrasast.gate import (
|
|
FALSE_POSITIVE_CEILING,
|
|
LANGUAGE_MANIFESTS,
|
|
RECALL_FLOOR,
|
|
detection_gate,
|
|
language_metrics,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("language", sorted(LANGUAGE_MANIFESTS))
|
|
def test_language_meets_recall_and_false_positive_goal(language: str) -> None:
|
|
metrics = language_metrics(LANGUAGE_MANIFESTS[language])
|
|
|
|
assert metrics.recall >= RECALL_FLOOR, (
|
|
f"{language} recall {metrics.recall:.2%} below {RECALL_FLOOR:.0%} ({metrics.matched}/{metrics.expected})"
|
|
)
|
|
assert metrics.false_positive_rate < FALSE_POSITIVE_CEILING, (
|
|
f"{language} false-positive rate {metrics.false_positive_rate:.2%} at or above "
|
|
f"{FALSE_POSITIVE_CEILING:.0%} ({metrics.false_positives}/{metrics.actual})"
|
|
)
|
|
|
|
|
|
def test_overall_detection_goal_is_met() -> None:
|
|
verdict = detection_gate({language: language_metrics(names) for language, names in LANGUAGE_MANIFESTS.items()})
|
|
assert verdict.passed, verdict.reasons
|