feat(real-world-vfc-slice): add labeled OpenSSL/Firefox/Chromium/curl pair corpus

Harvest 176 isolated vuln-vs-fix functions (109 CVEs, 352 training rows)
into benchmarks/pairs/vfc with provenance and a JSONL training manifest.
Score the slice as an honesty dashboard (ousast pairs --slice vfc);
keep the pair gate on the local fixtures only.
This commit is contained in:
Marius Ciepluch
2026-09-04 16:45:38 +02:00
parent e9f185da59
commit 0116f9bea6
372 changed files with 49209 additions and 49 deletions
+10 -4
View File
@@ -1,9 +1,9 @@
"""Pair-efficiency gate: local fixtures must fire on vuln and stay silent on fix.
The stage-1 smoke gate stays on cheat-sheet trees. This module measures
differential detection and only *fails CI* on the local slice. GitHub/SVEN
pairs are printed as the honesty dashboard (real VFC misses become improve
signals, they do not block merge).
differential detection and only *fails CI* on the local slice. GitHub/SVEN,
sast, and vfc pairs are honesty dashboards (`ousast pairs --slice ...`);
they are not evaluated here so a large vfc catalog cannot slow the merge gate.
python -m openultrasast.pair_gate
"""
@@ -36,7 +36,7 @@ def pair_gate(cases: Sequence[PairCase] | None = None) -> PairGateVerdict:
local = select_slice(catalog, "local")
if not local:
return PairGateVerdict(passed=False, reasons=("no local pairs in catalog",), result=evaluate_catalog(()))
result = evaluate_catalog(catalog)
result = evaluate_catalog(local)
local_metrics = result.per_slice.get("local")
reasons: list[str] = []
if local_metrics is None:
@@ -62,6 +62,12 @@ def print_pair_metrics(label: str, result: PairEvalResult) -> None:
f"vuln {metrics.vuln_recall:>6.2%} silent {metrics.specificity:>6.2%} "
f"Youden {metrics.youden:>+7.2%} labeled {metrics.labeled_recall:>6.2%}"
)
for slice_name, inner in result.scorers.items():
for scorer, metrics in inner.items():
print(
f" {slice_name:<8} scorer={scorer:<10} pair_pass {metrics.pair_pass_rate:>6.2%} "
f"({metrics.pair_correct}/{metrics.pairs}) Youden {metrics.youden:>+7.2%}"
)
for outcome in result.outcomes:
mark = "PASS" if outcome.pair_correct else "MISS" if not outcome.detected_vuln else "LEAK"
print(