feat(pair-corpus-honesty): profile gate scores only seeded and reviewed pairs

Task 8.2 (Req 9.3).

- improve/evolve.py: evaluate_profiles filters holdout pairs to
  GATING_TIERS before computing per-profile metrics, so advisory and
  title pairs are scored by `pairs` but never gate a round; a regression
  on reviewed pairs still rejects with profile_regression:<profile>.
- tests: a regression confined to title pairs is accepted; the same
  regression on reviewed pairs is rejected and the title pairs are
  excluded from the gated profile rather than outvoted.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/improve/evolve.py

@@ -57,10 +57,13 @@ def evaluate_profiles(
     pair_cases: Sequence[object],
     rules: tuple[PatternRule, ...],
 ) -> dict[str, dict[str, float]]:
-    """Pair-corpus metrics per provenance profile under ``rules`` (holdout split, no inventory sidecar)."""
-    from ..pairs import PairCase, evaluate_catalog
+    """Pair-corpus metrics per provenance profile under ``rules`` (holdout split, no inventory sidecar).
 
-    cases = [case for case in pair_cases if isinstance(case, PairCase)]
+    Only ``seeded`` and ``reviewed`` pairs gate (Req 9.3); ``advisory`` and ``title`` pairs are scored by ``pairs``, never here.
+    """
+    from ..pairs import GATING_TIERS, PairCase, evaluate_catalog, select_tier
+
+    cases = list(select_tier([case for case in pair_cases if isinstance(case, PairCase)], GATING_TIERS))
     if not cases:
         return {}
     result = evaluate_catalog(cases, ruleset=rules, inventory_sidecar=False)

tests/test_improve.py

@@ -173,7 +173,9 @@ def test_build_rule_signals_separates_miss_and_fp() -> None:
 # ---- pair-corpus-honesty: per-profile holdout clause (Req 7.2-7.4) ----------
 
 
-def _pair(tmp_path: Path, name: str, vuln_src: str, fixed_src: str, rule_id: str, sink: str, *, provenance: str):
+def _pair(
+    tmp_path: Path, name: str, vuln_src: str, fixed_src: str, rule_id: str, sink: str, *, provenance: str, review_tier: str = "reviewed"
+):
     from openultrasast.benchmark import ExpectedFinding
     from openultrasast.pairs import PairCase
 
@@ -204,6 +206,8 @@ def _pair(tmp_path: Path, name: str, vuln_src: str, fixed_src: str, rule_id: str
         fix_policy="silent",
         provenance=provenance,
         split="holdout",
+        review_tier=review_tier,
+        reviewer="test" if review_tier == "reviewed" else "",
     )
 
 
@@ -251,3 +255,41 @@ def test_round_accepts_when_profiles_hold_and_reports_small_profiles(tmp_path: P
     assert outcome.profile_regressions == []
     assert outcome.profiles_under_minimum == ["agent"]  # one pair: reported, not gated
     assert ledger.exists()
+
+
+def test_profile_gate_ignores_title_and_advisory_tiers(tmp_path: Path) -> None:
+    """Req 9.3: only seeded and reviewed pairs gate; a regression confined to title pairs never rejects a round."""
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text("def f(x):\n    return eval(x)\ndef g():\n    print('hello')\n")
+    ruleset_dir = _ruleset_dir(tmp_path, [_rule("good-eval", r"\beval\s*\("), _rule("noisy-print", r"\bprint\s*\(")])
+    _, manifest = _manifest(tmp_path, _expected("good-eval"))
+    ledger = tmp_path / "rule_policy.json"
+    journal = tmp_path / "journal.json"
+    title_pairs = [
+        _pair(
+            tmp_path,
+            f"t{i}",
+            "def h(y):\n    print(y)\n",
+            "def h(y):\n    return y\n",
+            "noisy-print",
+            "print",
+            provenance="agent",
+            review_tier="title",
+        )
+        for i in range(5)
+    ]
+    outcome = run_round(repo, manifest, ledger_path=ledger, journal_path=journal, ruleset_dir=ruleset_dir, pair_cases=title_pairs)
+    assert outcome.accepted and outcome.reason == "accepted"
+    assert outcome.profile_regressions == [] and "agent" not in outcome.per_profile_before
+    ledger.unlink()
+    journal.unlink()
+    reviewed_pairs = [
+        _pair(tmp_path, f"r{i}", "def h(y):\n    print(y)\n", "def h(y):\n    return y\n", "noisy-print", "print", provenance="agent")
+        for i in range(5)
+    ]
+    outcome = run_round(
+        repo, manifest, ledger_path=ledger, journal_path=journal, ruleset_dir=ruleset_dir, pair_cases=reviewed_pairs + title_pairs
+    )
+    assert not outcome.accepted and outcome.reason == "profile_regression:agent"
+    assert outcome.per_profile_before["agent"]["pairs"] == 5.0  # the five title pairs are scored elsewhere, never here