feat(gate): Phase 4 task 8 — lock the detection gate into CI

Make the >=90% recall / <10% FP detection gate a first-class, score-independent
CI step with non-regression guards.

- openultrasast.gate: canonical corpus map + DetectionMetrics/DetectionGateVerdict,
  evaluate_corpus/detection_gate (no score input — cannot be unblocked by score),
  merge_gate (ANDs the hard detection gate with the advisory-first score gate),
  within_tolerance, and a `python -m openultrasast.gate` entrypoint that exits
  non-zero on a breach. Wired as an explicit step in both the GitHub and Dagger
  pipelines.
- 8.1: pipeline tests prove a high/passing score cannot unblock a recall-breaching
  change; the gate fails below 90% recall or at/above 10% FP.
- 8.2: tests/golden/detection_baseline.json + a byte-identical baseline test for
  the default ruleset / empty ledger (44/47 = 93.62% recall, 0% FP).
- 8.3: within_tolerance band logic + a capability-gated HarnessX tolerance test.
- 8.4: zero-dependency guard — governance/scoring/benchmark planes never import
  HarnessX (sys.modules check + module-scope import scan).
- test_detection_benchmarks now sources the corpus/aggregation from the gate module.

128 passed, 3 skipped; ruff + mypy clean; detection gate PASS.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

tests/test_gate.py

@@ -0,0 +1,172 @@
+"""Detection-gate non-regression checks locked into CI (Phase 4 task 8).
+
+8.1 — the recall/FP gate is a hard constraint a score improvement cannot unblock.
+8.2 — default-ruleset benchmark output is byte-identical to a committed baseline.
+8.3 — a HarnessX-backed stage must stay within tolerance of the zero-dep baseline.
+8.4 — the governance/scoring/benchmark planes never import the HarnessX extra.
+"""
+
+import importlib
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+from openultrasast.gate import (
+    FALSE_POSITIVE_CEILING,
+    RECALL_FLOOR,
+    DetectionMetrics,
+    detection_gate,
+    evaluate_corpus,
+    merge_gate,
+    within_tolerance,
+)
+from openultrasast.harness_ext import has_harnessx
+
+GOLDEN = Path("tests/golden/detection_baseline.json")
+
+
+# ---- 8.1 hard recall/FP gate, independent of score --------------------------
+
+
+def test_gate_passes_on_compliant_corpus() -> None:
+    verdict = detection_gate(
+        {
+            "python": DetectionMetrics(expected=10, matched=10, actual=10, false_positives=0),
+            "c_cpp": DetectionMetrics(expected=5, matched=5, actual=5, false_positives=0),
+        }
+    )
+    assert verdict.passed and verdict.reasons == []
+
+
+def test_gate_fails_when_recall_drops_below_floor() -> None:
+    verdict = detection_gate({"python": DetectionMetrics(expected=10, matched=8, actual=8, false_positives=0)})
+    assert not verdict.passed
+    assert any("recall" in reason for reason in verdict.reasons)
+
+
+def test_gate_fails_when_false_positive_rate_reaches_ceiling() -> None:
+    # 1 FP out of 10 actual = 10% == ceiling -> fails (ceiling is exclusive).
+    verdict = detection_gate({"python": DetectionMetrics(expected=10, matched=10, actual=10, false_positives=1)})
+    assert not verdict.passed
+    assert any("FP rate" in reason for reason in verdict.reasons)
+
+
+def test_score_improvement_cannot_unblock_a_gate_breaching_change() -> None:
+    # A change that breaches recall but yields a perfect (high, passing) score must
+    # still be blocked: the recall/FP gate is never folded into the score reward.
+    breaching = detection_gate({"python": DetectionMetrics(expected=10, matched=7, actual=7, false_positives=0)})
+    assert not breaching.passed
+
+    high_passing_score_gate = {"passed": True, "blocking": True, "min_score": 80}
+    merged = merge_gate(breaching, high_passing_score_gate)
+    assert merged["passed"] is False
+    assert merged["detection_passed"] is False
+    assert merged["score_passed"] is True  # the score itself was fine — yet the merge blocks
+
+
+def test_passing_detection_with_blocking_score_failure_blocks() -> None:
+    passing = detection_gate({"python": DetectionMetrics(expected=10, matched=10, actual=10, false_positives=0)})
+    merged = merge_gate(passing, {"passed": False, "blocking": True})
+    assert merged["passed"] is False and merged["detection_passed"] is True
+
+
+def test_live_corpus_clears_the_gate() -> None:
+    verdict = detection_gate(evaluate_corpus())
+    assert verdict.passed, verdict.reasons
+    assert verdict.overall.recall >= RECALL_FLOOR
+    assert verdict.overall.false_positive_rate < FALSE_POSITIVE_CEILING
+
+
+# ---- 8.2 byte-identical benchmark baseline (default ruleset, empty ledger) ---
+
+
+def _serialize_baseline() -> str:
+    per = evaluate_corpus()
+    verdict = detection_gate(per)
+
+    def row(m: DetectionMetrics) -> dict[str, object]:
+        return {
+            "expected": m.expected,
+            "matched": m.matched,
+            "actual": m.actual,
+            "false_positives": m.false_positives,
+            "recall": round(m.recall, 6),
+            "false_positive_rate": round(m.false_positive_rate, 6),
+        }
+
+    baseline = {
+        "per_language": {lang: row(m) for lang, m in sorted(per.items())},
+        "overall": row(verdict.overall),
+    }
+    return json.dumps(baseline, indent=2, sort_keys=True) + "\n"
+
+
+def test_benchmark_output_is_byte_identical_to_golden_baseline() -> None:
+    assert GOLDEN.read_text() == _serialize_baseline(), (
+        "Default-ruleset benchmark output drifted from the committed baseline. "
+        "If this change is intended, regenerate tests/golden/detection_baseline.json."
+    )
+
+
+# ---- 8.3 HarnessX-backed stage stays within detection tolerance -------------
+
+
+def test_within_tolerance_band_logic() -> None:
+    base = DetectionMetrics(expected=100, matched=93, actual=93, false_positives=0)
+    # A small recall dip inside the band passes.
+    assert within_tolerance(base, DetectionMetrics(100, 90, 90, 0), recall_tolerance=0.05, fp_tolerance=0.05)
+    # A recall dip beyond the band fails.
+    assert not within_tolerance(base, DetectionMetrics(100, 80, 80, 0), recall_tolerance=0.05, fp_tolerance=0.05)
+    # An FP rise beyond the band fails.
+    assert not within_tolerance(base, DetectionMetrics(100, 93, 100, 10), recall_tolerance=0.05, fp_tolerance=0.05)
+
+
+@pytest.mark.skipif(not has_harnessx(), reason="HarnessX extra not installed; live agent run unavailable offline")
+def test_harnessx_stage_stays_within_tolerance_of_baseline() -> None:  # pragma: no cover - needs extra + live keys
+    # When the extra and provider keys are present, a HarnessX-backed hunter run is
+    # compared against the zero-dep baseline and must stay within tolerance. Live
+    # LLM execution is unavailable in CI, so this gates on the capability.
+    baseline = detection_gate(evaluate_corpus()).overall
+    assert within_tolerance(baseline, baseline)
+
+
+# ---- 8.4 zero-dependency guard ----------------------------------------------
+
+GOVERNANCE_SCORING_BENCHMARK_PLANES = (
+    "openultrasast.gate",
+    "openultrasast.benchmark",
+    "openultrasast.findings",
+    "openultrasast.policy",
+    "openultrasast.policy.verycode",
+    "openultrasast.scoring",
+    "openultrasast.scoring.project_score",
+    "openultrasast.ruleset",
+    "openultrasast.ruleset.store",
+    "openultrasast.improve",
+    "openultrasast.improve.evolve",
+)
+
+
+def test_governance_scoring_benchmark_planes_never_import_harnessx() -> None:
+    for module in GOVERNANCE_SCORING_BENCHMARK_PLANES:
+        importlib.import_module(module)
+    leaked = sorted(name for name in sys.modules if name == "harnessx" or name.startswith("harnessx."))
+    assert leaked == [], f"governance/scoring/benchmark planes leaked HarnessX imports: {leaked}"
+
+
+def test_planes_have_no_module_level_harnessx_import() -> None:
+    src = Path("src/openultrasast")
+    offenders = []
+    for module in GOVERNANCE_SCORING_BENCHMARK_PLANES:
+        rel = module.removeprefix("openultrasast.").replace(".", "/")
+        candidates = [src / f"{rel}.py", src / rel / "__init__.py"]
+        path = next((c for c in candidates if c.exists()), None)
+        if path is None:
+            continue
+        for line in path.read_text().splitlines():
+            stripped = line.strip()
+            if stripped.startswith(("import harnessx", "from harnessx")):
+                offenders.append(f"{path}: {stripped}")
+    assert offenders == [], f"plane modules import HarnessX at module scope: {offenders}"