fix(learning-harness): the round-3 review findings, remediated RED-first

Eight of nine group-3 tasks were rejected. Each defect was hiding a number,
and the numbers are why the loop was not learning:

- The fixed side was judged against the *vulnerable* side's function
  ranges. 191 of 227 pairs with a resolvable labeled span (84%) have
  different spans on the two sides, so a leak was recorded as silence,
  always in the direction that flatters Youden.
- Round zero measured every family with the generalist. `family:unknown`
  earns no credit against any real family, so every family but `unknown`
  baselined at a structural zero and every noise floor with it.
- The cross-family sweep counted absolute failures instead of negative
  flips. On a corpus where most pairs fail at round zero that rejects
  every round for damage it did not do. Unscorable rows voted too.
- K runs was declared and never threaded; `pairs --k-runs` was parsed and
  discarded, and its only test grepped `--help`.
- The meta-agent proposer was a stub, so `learning round` could never
  propose anything. It now composes MetaAgent.evolve against a scratch
  copy of the family directory, and `--proposals FILE` runs a recorded
  sequence offline.
- `refused_holdout` was a vocabulary word no code path could produce, and
  the covering test accepted either outcome.
- Cost was metered once, after everything had been paid for; a stage that
  raised left no record at all.
- The corrected lever number was published beside the earlier one rather
  than in place of it, and the test named for that property never checked.
- Refusals were `continue` statements: `Refusal.degradation()` existed and
  nothing called it. A train-split identical twin was reported as
  `split:train`, which is untrue.

Test repairs that each changed a number: two split tests measured an
`x = 1` fixture that teaches nothing whatever the rule says; two CLI
scoreboard tests asserted against all-zero output produced by a stub that
answers twice and puts its finding outside the labeled function.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/cli.py

@@ -7,7 +7,7 @@ from collections.abc import Callable, Sequence
 from dataclasses import dataclass, replace
 from pathlib import Path
 from time import perf_counter
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from . import tool_hunter
 from .benchmark import (
@@ -100,6 +100,10 @@ class ScanOutcome:
     exit_code: int
 
 
+if TYPE_CHECKING:
+    from .learning.proposer import Proposer
+
+
 def main(argv: list[str] | None = None) -> int:
     load_dotenv()
     parser = argparse.ArgumentParser(prog="ousast")
@@ -223,6 +227,12 @@ def main(argv: list[str] | None = None) -> int:
         if name == "round":
             command.add_argument("--family", required=True)
             command.add_argument("--cost-cap-usd", type=float, default=None)
+            command.add_argument(
+                "--proposals",
+                type=Path,
+                default=None,
+                help="a JSON list of proposals to try in order; without it the round asks the meta-agent",
+            )
         if name == "publish":
             command.add_argument("--measurements", type=Path, default=Path("benchmarks/measurements"))
             command.add_argument("--roadmap", type=Path, default=Path(".kiro/steering/roadmap.md"))
@@ -263,6 +273,7 @@ def main(argv: list[str] | None = None) -> int:
             pointers=args.pointers,
             loo=args.loo,
             loo_out=args.loo_out,
+            k_runs=args.k_runs,
         )
     if args.command == "mechanisms":
         return _mechanisms_export(args.catalog, args.slice, args.store, json_out=args.json)
@@ -954,6 +965,7 @@ def _pairs(
     pointers: bool = False,
     loo: bool = False,
     loo_out: Path | None = None,
+    k_runs: int | None = None,
 ) -> int:
     if not catalog.exists() or not catalog.is_file():
         raise SystemExit(f"pair catalog is not a file: {catalog}")
@@ -967,8 +979,10 @@ def _pairs(
         model = hunter_model or config.models.hunter or ""
         client = tool_hunter.resolve_hunter_client()
         if model and client is not None:
-            scan = make_hunter_scan(client, model)
-    result = evaluate_catalog(cases, hunter=scan, pointers=True if pointers else None)
+            from .learning.families import load_families
+
+            scan = make_hunter_scan(client, model, taxonomy=load_families())
+    result = evaluate_catalog(cases, hunter=scan, pointers=True if pointers else None, k_runs=k_runs)
     if json_out:
         print(json.dumps(result_payload(result), indent=2, sort_keys=True))
         return 0
@@ -1125,7 +1139,7 @@ def _run_family_detectors(
     suspicion here: the verifier writes a tag, and the evidence ladder is another boundary's to move.
     """
     del cases
-    from .learning.classify import classify_region, measure_classifier
+    from .learning.classify import classify_region
     from .learning.detectors import DetectorConfigError, Region, load_family_configs, run_region_detectors
     from .learning.endpoint import resolve_chat_endpoint, resolve_models
     from .learning.families import FamiliesError, load_families
@@ -1152,6 +1166,8 @@ def _run_family_detectors(
     model = resolve_models(config)[0]  # type: ignore[arg-type]
     findings: list[StaticFinding] = []
     per_family: dict[str, int] = {}
+    per_tier: dict[str, int] = {}
+    abstained = 0
     seen: set[str] = set()
     for hotspot in hotspots:
         path = str(getattr(hotspot, "path", ""))
@@ -1161,6 +1177,8 @@ def _run_family_detectors(
             continue
         seen.add(key)
         answer = classify_region(root, path, function=function, taxonomy=taxonomy, client=None)
+        per_tier[answer.tier] = per_tier.get(answer.tier, 0) + 1
+        abstained += 0 if answer.families else 1
         region = Region(path=path, function=str(function) if function else None, families=answer.families)
 
         def _run(region: Region = region) -> list[StaticFinding]:
@@ -1172,11 +1190,19 @@ def _run_family_detectors(
             verification = verifier_for(taxonomy.by_id(family_id)).verify(finding, root=root, language="python")
             findings.append(apply_verification(finding, verification))
             per_family[family_id] = per_family.get(family_id, 0) + 1
-    report = measure_classifier([], taxonomy, client=None)
     payload: dict[str, object] = {
         "taxonomy_version": taxonomy.version,
         "families": per_family,
-        "classifier": report.to_dict(),
+        # What the classifier did on *this* scan. Scoring it needs labels a maintainer stands behind, which a scan
+        # of someone else's repository does not have, so the block reports the routing rather than an accuracy.
+        "classifier": {
+            "regions": len(seen),
+            "per_tier": dict(sorted(per_tier.items())),
+            "abstained": abstained,
+            # Req 2.6/3.7: say out loud that no family declares a parent at this taxonomy version, so the
+            # partial credit for a parent-child relation never applied to any number here.
+            "hierarchical_credit": taxonomy.hierarchical,
+        },
         "round": None,
         "endpoint": endpoint.provider,
         "regions": len(seen),
@@ -1226,7 +1252,6 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
     from .learning.detectors import load_family_configs, write_default_configs
     from .learning.endpoint import resolve_chat_endpoint, resolve_models
     from .learning.journal import Archive, LearningJournal
-    from .learning.proposer import HarnessXProposer
     from .learning.rounds import MIN_K_RUNS, load_noise_floors, run_baseline, run_learning_round
     from .learning.scoring import aggregate
     from .tool_hunter import _SYSTEM_PROMPT
@@ -1259,7 +1284,9 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
         from .learning.rounds import score_case
 
         scores = [
-            score_case(case, factory(configs[_label(case)]), taxonomy=taxonomy, runs=k_runs) for case in cases if _label(case) in configs
+            score_case(case, factory(configs[_label(case, taxonomy)]), taxonomy=taxonomy, runs=k_runs, family=_label(case, taxonomy))
+            for case in cases
+            if _label(case, taxonomy) in configs
         ]
         families = {name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy).items()}
         if args.json:
@@ -1273,7 +1300,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
             cases,
             taxonomy=taxonomy,
             configs_dir=configs_dir,
-            scan=factory(configs.get("unknown") or next(iter(configs.values()))),
+            scan_factory=factory,
             model=model,
             out_dir=out,
             k_runs=k_runs,
@@ -1286,7 +1313,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
         family=args.family,
         taxonomy=taxonomy,
         configs_dir=configs_dir,
-        proposer=HarnessXProposer(configs_dir=configs_dir, model=model),
+        proposer=_proposer(args, configs_dir, model),
         scan_factory=factory,
         model=model,
         journal=journal,
@@ -1296,16 +1323,44 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
         cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd,
         minibatch=config.learning.minibatch,
         k_runs=k_runs,
+        # Every other family the corpus actually carries, so a change that reaches beyond its own directory is
+        # measured rather than assumed impossible (Req 9.5).
+        sweep_families=tuple(sorted({_label(case, taxonomy) for case in cases} & set(configs) - {args.family})),
     )
     print(f"learning round {record.round} {record.family}: {record.outcome} ({record.reason})")
     return 0
 
 
-def _label(case: PairCase) -> str:
-    for row in case.expected:
-        if row.family:
-            return str(row.family)
-    return "unknown"
+def _label(case: PairCase, taxonomy: object) -> str:
+    """The family that routes this pair: the maintainer's label when there is one, else the classifier's answer.
+
+    Reading only the declared label would send every unlabeled row to the generalist, whose findings earn no credit
+    against any real family, so a scoreboard built that way reports structural zeros (Req 2.1, 8.2)."""
+    from .learning.rounds import _family_of
+
+    return _family_of(case, taxonomy)  # type: ignore[arg-type]
+
+
+def _proposer(args: argparse.Namespace, configs_dir: Path, model: str) -> Proposer:
+    """`--proposals FILE` runs a recorded sequence offline; without it the round asks the meta-agent."""
+    from .learning.proposer import HarnessXProposer, Proposal, ScriptedProposer
+
+    path = getattr(args, "proposals", None)
+    if path is None:
+        return HarnessXProposer(configs_dir=configs_dir, model=model)
+    rows = json.loads(Path(path).read_text(encoding="utf-8"))
+    return ScriptedProposer(
+        [
+            Proposal(
+                hypothesis=str(row["hypothesis"]),
+                lever=row["lever"],
+                change=dict(row["change"]),
+                predicted_affected=tuple(row.get("predicted_affected", ())),
+                predicted_at_risk=tuple(row.get("predicted_at_risk", ())),
+            )
+            for row in rows
+        ]
+    )
 
 
 def _first_source(root: Path) -> str:

src/openultrasast/improve/evolve.py

@@ -52,6 +52,8 @@ class RoundOutcome:
     profiles_under_minimum: list[str] = field(default_factory=list)
     per_profile_before: dict[str, dict[str, float]] = field(default_factory=dict)
     per_profile_after: dict[str, dict[str, float]] = field(default_factory=dict)
+    # learning-harness Req 5.2: the candidates this round declined to learn from, by pair name.
+    degradations: list[dict[str, object]] = field(default_factory=list)
 
 
 MAX_PROFILES = 4
@@ -64,6 +66,10 @@ def evaluate_profiles(
     """Pair-corpus metrics per provenance profile under ``rules`` (holdout split, no inventory sidecar).
 
     Only ``seeded`` and ``reviewed`` pairs gate (Req 9.3); ``advisory`` and ``title`` pairs are scored by ``pairs``, never here.
+
+    Deliberately split-blind: this *measures* a ruleset, it never learns from a pair, so every pair is scored and the
+    teacher rule has nothing to say here. What the rule governs is the store these rules were built from
+    (`semantic.seed`, `semantic.loo`), and a filter here would only shrink the evidence (learning-harness Req 5.1).
     """
     from ..pairs import GATING_TIERS, PairCase, evaluate_catalog, select_tier, select_vendored
 
@@ -84,7 +90,10 @@ def evaluate_profiles(
 
 
 def evaluate_mechanism_profiles(pair_cases: Sequence[object], store_path: Path) -> dict[str, dict[str, float]]:
-    """Per-profile pair metrics of variant search with the given scan-time store over the holdout pairs (vendored, gating tiers)."""
+    """Per-profile pair metrics of variant search with the given scan-time store over the holdout pairs (vendored, gating tiers).
+
+    Split-blind for the same reason as `evaluate_profiles`: the store is the thing the teacher rule governs, and this
+    function only reads it (learning-harness Req 5.1)."""
     from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored
     from ..semantic.loo import score_pair_with_store
     from ..semantic.mechanisms import MechanismStore
@@ -110,7 +119,12 @@ def evaluate_mechanism_profiles(pair_cases: Sequence[object], store_path: Path)
     return result
 
 
-def propose_mechanism_edits(pair_cases: Sequence[object], candidates: object, store_path: Path) -> list[MechanismEdit]:
+def propose_mechanism_edits(
+    pair_cases: Sequence[object],
+    candidates: object,
+    store_path: Path,
+    degradations: list[dict[str, object]] | None = None,
+) -> list[MechanismEdit]:
     """Req 5.3: admit a candidate that recovers a currently missed holdout pair without leaking; retract an admitted leaker."""
     from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored
     from ..semantic.loo import score_pair_with_store
@@ -139,8 +153,13 @@ def propose_mechanism_edits(pair_cases: Sequence[object], candidates: object, st
     for record in corpus_mechanisms(candidates.load()):  # type: ignore[attr-defined]
         if record.id in admitted or not missed:
             continue
-        if refuse_if_holdout(record.pairs, cases_all) is not None:
-            continue  # learning-harness Req 5.2: a shape a holdout pair taught can never recover that pair
+        refusal = refuse_if_holdout(record.pairs, cases_all)
+        if refusal is not None:
+            # learning-harness Req 5.2: a shape a holdout pair taught can never recover that pair, and the run
+            # says which pairs it was rather than showing a shorter list with no reason.
+            if degradations is not None:
+                degradations.append(refusal.degradation())
+            continue
         with tempfile.TemporaryDirectory(prefix="ousast-lever-") as scratch:
             trial = MechanismStore(Path(scratch) / "trial.jsonl")
             trial.append(record)
@@ -278,6 +297,7 @@ def run_round(
     edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked)
     # corpus-seeded-mechanisms Req 5: the `mechanisms` lever proposes from the holdout pairs, gated like every other edit.
     mechanism_edits: list[MechanismEdit] = []
+    refusals: list[dict[str, object]] = []
     candidates_store = None
     if mechanism_candidates is not None and mechanism_store is not None and pair_cases:
         from ..semantic.mechanisms import MechanismStore
@@ -286,12 +306,12 @@ def run_round(
         mechanism_edits = (
             list(scripted_mechanism_edits)
             if scripted_mechanism_edits is not None
-            else propose_mechanism_edits(pair_cases, candidates_store, mechanism_store)
+            else propose_mechanism_edits(pair_cases, candidates_store, mechanism_store, degradations=refusals)
         )
         blocked_mechanisms = {edit.key() for edit in mechanism_edits} & blocked
         mechanism_edits = [edit for edit in mechanism_edits if edit.key() not in blocked_mechanisms]
     if not edits and not mechanism_edits:
-        return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before)
+        return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before, degradations=refusals)
 
     validator = EvolveValidator()
     try:
@@ -314,6 +334,7 @@ def run_round(
             before=before,
             after=before,
             mechanism_edits=mechanism_edits,
+            degradations=refusals,
         )
 
     candidate = edits_to_ledger(edits, current_ledger)
@@ -321,7 +342,9 @@ def run_round(
         _load_with(ruleset_dir, candidate)  # replay smoke gate: candidate ruleset must boot
     except Exception as exc:  # noqa: BLE001 — any boot failure rejects the round
         _record(journal_path, round_index, edits, "rejected", before, before, f"replay_failed: {exc}")
-        return _outcome(round_index, accepted=False, reason="replay_failed", edits=edits, before=before, after=before)
+        return _outcome(
+            round_index, accepted=False, reason="replay_failed", edits=edits, before=before, after=before, degradations=refusals
+        )
 
     after_findings, after_rules = _scan(target, ruleset_dir, candidate, policy)
     after = _evaluate(target, manifest, after_findings, after_rules, policy)
@@ -395,7 +418,14 @@ def run_round(
         candidates=candidates_store,
     )
     outcome = _outcome(
-        round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after, mechanism_edits=mechanism_edits
+        round_index,
+        accepted=accepted,
+        reason=reason,
+        edits=edits,
+        before=before,
+        after=after,
+        mechanism_edits=mechanism_edits,
+        degradations=refusals,
     )
     return RoundOutcome(
         **{
@@ -513,6 +543,7 @@ def _outcome(
     before: _Eval,
     after: _Eval,
     mechanism_edits: list[MechanismEdit] | None = None,
+    degradations: list[dict[str, object]] | None = None,
 ) -> RoundOutcome:
     return RoundOutcome(
         round=round_index,
@@ -520,6 +551,7 @@ def _outcome(
         reason=reason,
         edits=edits,
         mechanism_edits=list(mechanism_edits or []),
+        degradations=list(degradations or []),
         recall_before=before.recall,
         recall_after=after.recall,
         fp_before=before.fp_rate,

src/openultrasast/learning/proposer.py

@@ -12,7 +12,9 @@ construction rather than by promise, and the verifier code and the corpus stay o
 from __future__ import annotations
 
 import json
-from collections.abc import Mapping, Sequence
+import shutil
+import tempfile
+from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Literal, Protocol
@@ -25,7 +27,7 @@ from .scoring import PairFamilyScore
 Lever = Literal["tool", "checklist", "counterexample", "memory", "prompt"]
 LEVERS: tuple[Lever, ...] = ("tool", "checklist", "counterexample", "memory", "prompt")
 # The only files a round may write, one per lever. Anything else is a proposal reaching beyond its family.
-EDITABLE_FILES = {
+EDITABLE_FILES: dict[Lever, str] = {
     "tool": "family.toml",
     "checklist": "checklist.md",
     "counterexample": "counterexamples.jsonl",
@@ -159,31 +161,109 @@ class ScriptedProposer:
         return self.proposals[self.used - 1]
 
 
+MetaAgentCall = Callable[..., str]
+
+
 @dataclass
 class HarnessXProposer:
-    """The meta-agent, with write access limited to the family directory it is targeting."""
+    """The meta-agent, with write access limited to a copy of the family directory it is targeting.
+
+    It never edits the live tree. The round snapshots the family directory *after* a proposal exists, so a proposer
+    that wrote in place would have changed the configuration before there was anything to restore, and a rejection
+    could not put it back. The agent works in a scratch copy and the proposal is the diff of that copy, which also
+    means the one-file rule and the lever whitelist are enforced on what the agent actually did rather than on what
+    it claimed (Req 9.1, 9.2, 5.4).
+    """
 
     configs_dir: Path
     model: str
     reason: str = ""
+    agent: MetaAgentCall | None = None  # injected in tests; None composes MetaAgent behind harness_ext
 
     def write_roots(self, family: str) -> tuple[Path, ...]:
         return write_roots(self.configs_dir, family)
 
     def propose(self, facts: FailureFacts) -> Proposal | None:
-        if not has_harnessx():
-            self.reason = "harnessx_unavailable"
+        call = self.agent
+        if call is None:
+            if not has_harnessx():
+                self.reason = "harnessx_unavailable"
+                return None
+            try:
+                call = _meta_agent_call()
+            except Exception:  # noqa: BLE001 — a proposer that cannot start proposes nothing and says so
+                self.reason = "harnessx_unavailable"
+                return None
+        family_dir = self.configs_dir / facts.family
+        with tempfile.TemporaryDirectory(prefix="ousast-propose-") as scratch:
+            workspace = Path(scratch) / facts.family
+            shutil.copytree(family_dir, workspace)
+            before = _directory_text(workspace)
+            try:
+                hypothesis = call(workspace=workspace, facts=facts, model=self.model)
+            except Exception as exc:  # noqa: BLE001 — an agent that failed proposed nothing, and the round says why
+                self.reason = f"meta_agent_failed: {type(exc).__name__}"
+                return None
+            after = _directory_text(workspace)
+        changed = sorted(name for name in set(before) | set(after) if before.get(name) != after.get(name))
+        if not changed:
+            self.reason = "no_change_proposed"
             return None
-        try:
-            from ..harness_ext import require_harnessx
+        if len(changed) > 1:
+            self.reason = f"a round changes one file, not {len(changed)}: {', '.join(changed)}"
+            return None
+        name = changed[0]
+        lever: Lever | None = next((key for key, filename in EDITABLE_FILES.items() if filename == name), None)
+        if lever is None or name not in after:
+            self.reason = f"{name!r} is not a file any lever writes"
+            return None
+        self.reason = ""
+        return Proposal(
+            hypothesis=str(hypothesis or "").strip() or f"meta-agent edit to {name}",
+            lever=lever,
+            change={name: after[name]},
+            predicted_affected=tuple(item.pair for item in facts.misses),
+            predicted_at_risk=tuple(item.pair for item in facts.leaks),
+        )
 
-            require_harnessx()
-        except Exception:  # noqa: BLE001 — a proposer that cannot start proposes nothing and says so
-            self.reason = "harnessx_unavailable"
-            return None
-        self.reason = "harnessx_proposer_not_wired"  # the evolve call lands with the live rounds (task 5.3)
-        del facts
-        return None
+
+def _directory_text(directory: Path) -> dict[str, str]:
+    """Every readable file directly inside ``directory``, by name. Nested paths are not a lever's business."""
+    out: dict[str, str] = {}
+    for path in sorted(directory.iterdir()) if directory.is_dir() else []:
+        if path.is_file():
+            try:
+                out[path.name] = path.read_text(encoding="utf-8")
+            except (OSError, UnicodeDecodeError):
+                continue
+    return out
+
+
+def _meta_agent_call() -> MetaAgentCall:
+    """Compose `MetaAgent.evolve` behind `harness_ext`, writing only inside the scratch copy.
+
+    `evolve` is a coroutine and returns a HarnessX config path; what this proposer takes from it is the state of the
+    workspace afterwards, so the return value is only used for the hypothesis line.
+    """
+    from ..harness_ext import require_harnessx
+
+    require_harnessx()
+
+    def call(*, workspace: Path, facts: FailureFacts, model: str) -> str:
+        import asyncio
+
+        from harnessx.meta_harness.agent import MetaAgent  # type: ignore[import-not-found]
+
+        trajectories = workspace.parent / "trajectories"
+        trajectories.mkdir(parents=True, exist_ok=True)
+        (trajectories / "facts.json").write_text(facts_prompt(facts), encoding="utf-8")
+        agent = MetaAgent(inner_model=model, allowed_write_roots=(workspace,))  # type: ignore[arg-type]
+        output = workspace.parent / "meta-out"
+        output.mkdir(parents=True, exist_ok=True)
+        asyncio.run(agent.evolve(current_config=workspace, trajectories_dir=trajectories, output_dir=output))
+        return f"meta-agent round for {facts.family}"
+
+    return call
 
 
 def _fact(score: PairFamilyScore, other: int) -> PairFact:

src/openultrasast/learning/publish.py

@@ -12,6 +12,7 @@ rather than appended, so the file cannot accumulate stale numbers beside fresh o
 from __future__ import annotations
 
 import json
+import re
 from collections.abc import Mapping
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
@@ -69,6 +70,7 @@ def publish(*, learning_dir: Path, measurements_dir: Path, roadmap: Path, date:
         )
         artifacts.append(artifact)
     _replace_section(roadmap, table)
+    _supersede(roadmap)
     return PublishReport(command=COMMAND, table=table, families=families, models=tuple(sorted(models)), artifacts=tuple(artifacts))
 
 
@@ -169,6 +171,32 @@ def _cost(cost: float, correct: int) -> str:
     return f"{cost / correct:.2f}" if cost and correct else "-"
 
 
+# Claims the split correction supersedes, as they were written before holdout pairs stopped teaching. Req 5.3 says
+# the corrected number is published *in place of* the earlier one; a document that carries both asserts two numbers
+# for one measurement and the reader has no way to tell which run is current.
+_SUPERSEDED = (
+    re.compile(r"human holdout profile pair_correct \d+ -> \d+ of \d+, Youden [-+]?\d+ -> [-+]?[\d.]+(?:, round \d+ `[^`]+`)?\."),
+    re.compile(r"The lever recovered \d+ of \d+ holdout pairs at Youden [-+][\d.]+\."),
+)
+_SUPERSEDED_NOTE = (
+    "superseded by the learning-harness section below: that number was measured with holdout pairs teaching "
+    "(see `benchmarks/measurements/2026-09-06-mechanism-lever-split.json`)."
+)
+
+
+def _supersede(roadmap: Path) -> None:
+    """Rewrite every superseded lever claim to point at the regenerated section. Idempotent: the pattern is gone
+    afterwards, so a second run changes nothing."""
+    if not roadmap.is_file():
+        return
+    text = roadmap.read_text(encoding="utf-8")
+    updated = text
+    for pattern in _SUPERSEDED:
+        updated = pattern.sub(_SUPERSEDED_NOTE, updated)
+    if updated != text:
+        roadmap.write_text(updated, encoding="utf-8")
+
+
 def _replace_section(roadmap: Path, table: str) -> None:
     body = f"{BEGIN}\n\n## Classified detector numbers\n\n{table.rstrip()}\n\n{END}\n"
     text = roadmap.read_text(encoding="utf-8") if roadmap.is_file() else ""

src/openultrasast/learning/rounds.py

@@ -68,6 +68,9 @@ class BaselineReport:
     slices: tuple[str, ...]
     floors: dict[str, NoiseFloor] = field(default_factory=dict)
     metrics: dict[str, dict[str, object]] = field(default_factory=dict)
+    # `<slice>/<family>`: the budgets stay per family, but vibe-py and agent-vfc are different worlds and one
+    # pooled number hides which of them moved (Req 8.4).
+    per_slice: dict[str, dict[str, object]] = field(default_factory=dict)
     scores: tuple[PairFamilyScore, ...] = ()
     cost_usd: float = 0.0
 
@@ -80,6 +83,7 @@ class BaselineReport:
             "cost_usd": self.cost_usd,
             "floors": {name: floor.to_dict() for name, floor in sorted(self.floors.items())},
             "metrics": {name: dict(block) for name, block in sorted(self.metrics.items())},
+            "per_slice": {name: dict(block) for name, block in sorted(self.per_slice.items())},
         }
 
 
@@ -88,7 +92,7 @@ def run_baseline(
     *,
     taxonomy: FamilyTaxonomy,
     configs_dir: Path,
-    scan: Scan,
+    scan_factory: Callable[[FamilyConfig], Scan],
     model: str,
     out_dir: Path,
     slices: Sequence[str] = DEFAULT_SLICES,
@@ -96,13 +100,29 @@ def run_baseline(
     prompt: str | None = None,
     spent_usd: Callable[[], float] | None = None,
 ) -> BaselineReport:
-    """Clone one detector into every family, measure each family against itself, and write the artifacts."""
-    from ..tool_hunter import _SYSTEM_PROMPT
+    """Clone one detector into every family, measure each family **with its own configuration**, write the artifacts.
 
+    The factory is the point. One shared `scan` would score every family with one detector, and since the generalist
+    tags its findings `family:unknown` — which earns no credit against any real family, by the taxonomy's own
+    decision that abstention is not a partial answer — every family but `unknown` would baseline at a structural
+    zero and the noise floors would all be nil (Req 8.2).
+    """
+    from ..tool_hunter import _SYSTEM_PROMPT
+    from .detectors import load_family_configs
+
+    if k_runs < MIN_K_RUNS:
+        raise ValueError(f"round zero scores at least three runs per pair; {k_runs} was asked for")
     if not configs_dir.exists():
         write_default_configs(configs_dir, taxonomy, prompt=prompt or _SYSTEM_PROMPT, version="0")
+    configs = load_family_configs(configs_dir, taxonomy)
     selected = [case for case in cases if case.slice in set(slices)]
-    scores = [score_case(case, scan, taxonomy=taxonomy, runs=k_runs) for case in selected]
+    scores = []
+    for case in selected:
+        name = _family_of(case, taxonomy)
+        config = configs.get(name)
+        if config is None:
+            continue  # a family with no configuration is not measured rather than measured with someone else's
+        scores.append(score_case(case, scan_factory(config), taxonomy=taxonomy, runs=k_runs, family=name))
     metrics = aggregate(scores, taxonomy=taxonomy)
     floors = {name: _floor(name, block, [item for item in scores if item.family == name], k_runs) for name, block in metrics.items()}
     report = BaselineReport(
@@ -112,6 +132,7 @@ def run_baseline(
         slices=tuple(slices),
         floors=floors,
         metrics={name: block.to_dict() for name, block in metrics.items()},
+        per_slice={name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, per_slice=True).items()},
         scores=tuple(scores),
         cost_usd=spent_usd() if spent_usd is not None else 0.0,
     )
@@ -141,9 +162,12 @@ def score_case(case: PairCase, scan: Scan, *, taxonomy: FamilyTaxonomy, runs: in
         vuln_root = _materialize_side(Path(scratch) / "vuln", case, side="vuln")
         fix_root = _materialize_side(Path(scratch) / "fixed", case, side="fixed")
         ranges = _overlay_scan(vuln_root).ranges
+        fixed_ranges = _overlay_scan(fix_root).ranges
         vuln_runs = [scan(vuln_root) for _ in range(max(runs, 1))]
         fix_runs = [scan(fix_root) for _ in range(max(runs, 1))]
-    return score_pair_family(case, name, vuln_runs, fix_runs, ranges=ranges, taxonomy=taxonomy, parse_ok=bool(ranges))
+    return score_pair_family(
+        case, name, vuln_runs, fix_runs, ranges=ranges, taxonomy=taxonomy, parse_ok=bool(ranges), fixed_ranges=fixed_ranges
+    )
 
 
 def compare_baselines(out_dir: Path) -> dict[str, dict[str, dict[str, object]]]:
@@ -249,11 +273,10 @@ def run_learning_round(
     real changes. The proposer never sees a held-out pair, and the family directory is snapshotted before
     the change so any outcome other than acceptance leaves the tree exactly as it was found.
     """
-    from ..redaction import redact_secrets
     from .acceptance import DirectorySnapshot, bump_version, decide, within_budget
     from .detectors import load_family_configs
     from .proposer import apply_proposal, build_failure_facts, refuse_proposal
-    from .split import is_teacher
+    from .split import is_teacher, refuse_if_holdout
 
     if k_runs < MIN_K_RUNS:
         raise ValueError(f"a stage scores at least three runs per pair; {k_runs} was asked for")
@@ -268,33 +291,73 @@ def run_learning_round(
     train = [case for case in cases if is_teacher(case) and _family_of(case, taxonomy) == family]
     holdout = [case for case in cases if case.split == "holdout" and _family_of(case, taxonomy) == family]
     batch = train[:minibatch]
+    spent = 0.0
 
     def _score(rows: Sequence[PairCase], scan: Scan, name: str) -> list[PairFamilyScore]:
         return [score_case(case, scan, taxonomy=taxonomy, runs=k_runs, family=name) for case in rows]
 
+    def _over_cap() -> bool:
+        """Req 9.8: cost is read after every stage, so a round stops rather than finishing and then complaining."""
+        nonlocal spent
+        spent = spent_usd() if spent_usd is not None else 0.0
+        return not within_budget(spent, cost_cap_usd)
+
     before_scan = scan_factory(config)
     before_train = _score(batch, before_scan, family)
     before_hold = _score(holdout, before_scan, family)
+    record = _round_record(round_number, family, None, taxonomy, config)
+    if _over_cap():
+        return _finish(
+            journal, directory, record, outcome="reverted_cost", reason="cost_cap_exceeded", cost=spent, trajectories=trajectories
+        )
     facts = build_failure_facts(before_train, family=family, config=config, journal=journal, train_pairs={case.name for case in train})
     proposal = proposer.propose(facts)  # type: ignore[attr-defined]
     record = _round_record(round_number, family, proposal, taxonomy, config)
     if proposal is None:
-        return _finish(journal, directory, record, outcome="rejected", reason="no_proposal", trajectories=trajectories)
+        return _finish(journal, directory, record, outcome="rejected", reason="no_proposal", cost=spent, trajectories=trajectories)
     refusal = refuse_proposal(proposal, config)
     if refusal is not None:
-        return _finish(journal, directory, record, outcome="rejected", reason=refusal, proposal=proposal, trajectories=trajectories)
+        return _finish(
+            journal, directory, record, outcome="rejected", reason=refusal, proposal=proposal, cost=spent, trajectories=trajectories
+        )
+    leak = refuse_if_holdout(_pairs_named_by(proposal, cases), cases)
+    if leak is not None:
+        # Req 5.2: a change a holdout pair taught, or that names one, is refused by name rather than measured.
+        return _finish(
+            journal,
+            directory,
+            record,
+            outcome="refused_holdout",
+            reason=leak.degradation()["detail"],  # type: ignore[arg-type]
+            proposal=proposal,
+            cost=spent,
+            trajectories=trajectories,
+        )
+    # The sweep's "before" is measured with the unchanged configurations, so what it later reports is what this
+    # round flipped and not what was already failing (Req 9.5).
+    before_sweep = _sweep_scores(cases, taxonomy, configs, scan_factory, sweep_families, family, k_runs)
+    if _over_cap():
+        return _finish(
+            journal,
+            directory,
+            record,
+            outcome="reverted_cost",
+            reason="cost_cap_exceeded",
+            proposal=proposal,
+            cost=spent,
+            trajectories=trajectories,
+        )
     snapshot = DirectorySnapshot.of(configs_dir / family)
     apply_proposal(proposal, configs_dir / family)
+    version = str(int(config.version) + 1) if config.version.isdigit() else f"{config.version}+1"
     try:
         after_config = load_family_configs(configs_dir, taxonomy)[family]
         after_scan = scan_factory(after_config)
         after_train = _score(batch, after_scan, family)
         after_hold = _score(holdout, after_scan, family)
-        sweep, flipped_unpredicted = _sweep(cases, taxonomy, configs, scan_factory, sweep_families, family, k_runs)
-        cost = spent_usd() if spent_usd is not None else 0.0
-        train_delta = _delta(before_train, after_train)
-        holdout_delta = _delta(before_hold, after_hold)
-        if not within_budget(cost, cost_cap_usd):
+        for score in after_train + after_hold:
+            archive.record(family, version, score)  # type: ignore[attr-defined]
+        if _over_cap():
             snapshot.restore()
             return _finish(
                 journal,
@@ -303,17 +366,34 @@ def run_learning_round(
                 outcome="reverted_cost",
                 reason="cost_cap_exceeded",
                 proposal=proposal,
-                cost=cost,
+                cost=spent,
                 scores=after_train + after_hold,
                 trajectories=trajectories,
             )
-        verdict = decide(target=family, train_delta=train_delta, holdout_delta=holdout_delta, sweep=sweep, floors=floors)  # type: ignore[arg-type]
+        after_sweep = _sweep_scores(cases, taxonomy, configs, scan_factory, sweep_families, family, k_runs)
+        sweep, flipped_unpredicted = _negative_flips(before_sweep, after_sweep)
+        train_delta = _delta(before_train, after_train)
+        holdout_delta = _delta(before_hold, after_hold)
         flipped = _flipped(before_train + before_hold, after_train + after_hold)
         attribution = Attribution(
             flipped_predicted=len(flipped),
             flipped_unpredicted=len(flipped_unpredicted),
             precision=len(flipped) / (len(flipped) + len(flipped_unpredicted)) if flipped or flipped_unpredicted else 0.0,
         )
+        if _over_cap():
+            snapshot.restore()
+            return _finish(
+                journal,
+                directory,
+                record,
+                outcome="reverted_cost",
+                reason="cost_cap_exceeded",
+                proposal=proposal,
+                cost=spent,
+                scores=after_train + after_hold,
+                trajectories=trajectories,
+            )
+        verdict = decide(target=family, train_delta=train_delta, holdout_delta=holdout_delta, sweep=sweep, floors=floors)  # type: ignore[arg-type]
         if not verdict.accepted:
             snapshot.restore()
             return _finish(
@@ -323,7 +403,7 @@ def run_learning_round(
                 outcome="rejected",
                 reason=verdict.reason,
                 proposal=proposal,
-                cost=cost,
+                cost=spent,
                 train_delta=train_delta,
                 holdout_delta=holdout_delta,
                 sweep=sweep,
@@ -333,10 +413,7 @@ def run_learning_round(
                 scores=after_train + after_hold,
                 trajectories=trajectories,
             )
-        version = str(int(config.version) + 1) if config.version.isdigit() else f"{config.version}+1"
         bump_version(configs_dir / family, version)
-        for score in after_train + after_hold:
-            archive.record(family, version, score)  # type: ignore[attr-defined]
         return _finish(
             journal,
             directory,
@@ -344,7 +421,7 @@ def run_learning_round(
             outcome="accepted",
             reason=verdict.reason,
             proposal=proposal,
-            cost=cost,
+            cost=spent,
             train_delta=train_delta,
             holdout_delta=holdout_delta,
             sweep=sweep,
@@ -356,13 +433,37 @@ def run_learning_round(
             trajectories=trajectories,
         )
     except Exception:
+        # A stage that raises has already spent money and already changed the tree. Both facts belong in the
+        # journal before the exception leaves, or the next round starts from a state nothing describes.
         snapshot.restore()
+        _finish(
+            journal,
+            directory,
+            record,
+            outcome="reverted",
+            reason="stage_failed",
+            proposal=proposal,
+            cost=spent,
+            trajectories=trajectories,
+        )
         raise
-    finally:
-        del redact_secrets
 
 
-def _sweep(
+def _pairs_named_by(proposal: Proposal, cases: Sequence[PairCase]) -> tuple[str, ...]:
+    """Every corpus pair this proposal names, in its predictions or in the file it writes.
+
+    A memory entry that quotes a held-out pair is the loop learning the answer instead of the shape, and it looks
+    exactly like a legitimate hard negative until the holdout number is read as if it meant something."""
+    haystack = "\n".join([*proposal.predicted_affected, *proposal.predicted_at_risk, *proposal.change.values()])
+    named = []
+    for case in cases:
+        name = str(getattr(case, "name", ""))
+        if name and re.search(rf"(?<![\w-]){re.escape(name)}(?![\w-])", haystack):
+            named.append(name)
+    return tuple(sorted(set(named)))
+
+
+def _sweep_scores(
     cases: Sequence[PairCase],
     taxonomy: FamilyTaxonomy,
     configs: Mapping[str, FamilyConfig],
@@ -370,23 +471,34 @@ def _sweep(
     families: Sequence[str],
     target: str,
     k_runs: int,
-) -> tuple[dict[str, int], list[str]]:
-    """Every other family's held-out pairs, so a change cannot help one family by quietly breaking another."""
-    flips: dict[str, int] = {}
-    unpredicted: list[str] = []
+) -> dict[str, dict[str, str]]:
+    """Every other family's held-out pairs, family by family, as `{family: {pair: outcome}}`.
+
+    Called once before the change and once after it. Scoring both sides is the only way to say which pairs *this*
+    round broke: on a real corpus most pairs are failing already, and counting failures would reject every round
+    for damage it did not do (Req 9.5). Unscorable rows are dropped here rather than counted as failures (Req 4.1).
+    """
+    outcomes: dict[str, dict[str, str]] = {}
     for name in families:
         if name == target or name not in configs:
             continue
         rows = [case for case in cases if case.split == "holdout" and _family_of(case, taxonomy) == name]
         scan = scan_factory(configs[name])
-        broken = [
-            item
-            for item in (score_case(case, scan, taxonomy=taxonomy, runs=k_runs, family=name) for case in rows)
-            if item.outcome != "pair_correct"
-        ]
-        flips[name] = len(broken)
-        unpredicted.extend(item.pair for item in broken)
-    return flips, unpredicted
+        scored = [score_case(case, scan, taxonomy=taxonomy, runs=k_runs, family=name) for case in rows]
+        outcomes[name] = {item.pair: item.outcome for item in scored if item.outcome != "unscorable"}
+    return outcomes
+
+
+def _negative_flips(before: Mapping[str, Mapping[str, str]], after: Mapping[str, Mapping[str, str]]) -> tuple[dict[str, int], list[str]]:
+    """Pairs that were right before this round and are not right after it, per family and by name."""
+    flips: dict[str, int] = {}
+    broken: list[str] = []
+    for name, was in before.items():
+        now = after.get(name, {})
+        lost = sorted(pair for pair, outcome in was.items() if outcome == "pair_correct" and now.get(pair) != "pair_correct")
+        flips[name] = len(lost)
+        broken.extend(lost)
+    return flips, broken
 
 
 def _delta(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> float:
@@ -400,8 +512,9 @@ def _delta(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore])
 
 
 def _flipped(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> list[str]:
+    """Pairs this round turned right, by name and in a stable order."""
     was = {item.pair: item.outcome for item in before}
-    return [item.pair for item in after if item.outcome == "pair_correct" and was.get(item.pair) not in {"pair_correct", None}]
+    return sorted(item.pair for item in after if item.outcome == "pair_correct" and was.get(item.pair) not in {"pair_correct", None})
 
 
 def _round_record(

src/openultrasast/learning/scoring.py

@@ -58,14 +58,21 @@ def score_pair_family(
     taxonomy: FamilyTaxonomy,
     parse_ok: bool = True,
     entry_points: Sequence[str] | None = None,
+    fixed_ranges: FunctionRanges | None = None,
 ) -> PairFamilyScore:
-    """Score one pair for one family over K runs of the detector on each side."""
+    """Score one pair for one family over K runs of the detector on each side.
+
+    ``fixed_ranges`` is the *fixed* side's parse. A fix that adds a guard above the handler moves its first line,
+    so judging the fixed side against the vulnerable side's span reads a finding inside the fixed handler as
+    outside it: a leak recorded as silence, and always in the direction that flatters the number.
+    """
     name = str(getattr(case, "name", "?"))
     slice_name = str(getattr(case, "slice", "") or "")
     reason = unscorable_reason(case, parse_ok=parse_ok, ranges=ranges, entry_points=entry_points, family=family)
     if reason is not None:
         return PairFamilyScore(pair=name, family=family, slice=slice_name, runs=(), outcome="unscorable", unscorable_reason=reason)
     spans = _spans(case, ranges)
+    fixed_spans = _spans(case, fixed_ranges) if fixed_ranges is not None else spans
     outcomes: list[FamilyOutcome] = []
     other_vuln = other_fixed = fabricated_vuln = fabricated_fixed = 0
     rung: Rung = "suspicion"
@@ -73,9 +80,9 @@ def score_pair_family(
         vuln = list(vuln_runs[index]) if index < len(vuln_runs) else []
         fixed = list(fixed_runs[index]) if index < len(fixed_runs) else []
         hits_vuln = _hits(vuln, family, spans, taxonomy)
-        hits_fixed = _hits(fixed, family, spans, taxonomy)
+        hits_fixed = _hits(fixed, family, fixed_spans, taxonomy)
         other_vuln += _others(vuln, family, spans, taxonomy)
-        other_fixed += _others(fixed, family, spans, taxonomy)
+        other_fixed += _others(fixed, family, fixed_spans, taxonomy)
         fabricated_vuln += _fabricated(vuln, taxonomy)
         fabricated_fixed += _fabricated(fixed, taxonomy)
         rung = _best_rung(rung, hits_vuln)

src/openultrasast/pairs.py

@@ -417,12 +417,18 @@ def _review_tier(item: dict[str, object], slice_name: str, name: str) -> tuple[s
     return tier, "" if pending else reviewer
 
 
-def evaluate_pair(case: PairCase, *, hunter: HunterScan | None = None, ruleset: tuple[PatternRule, ...] | None = None) -> PairOutcome:
+def evaluate_pair(
+    case: PairCase,
+    *,
+    hunter: HunterScan | None = None,
+    ruleset: tuple[PatternRule, ...] | None = None,
+    k_runs: int | None = None,
+) -> PairOutcome:
     with tempfile.TemporaryDirectory(prefix="ousast-pair-") as scratch:
         vuln_root = _materialize_side(Path(scratch) / "vuln", case, side="vuln")
         fix_root = _materialize_side(Path(scratch) / "fixed", case, side="fixed")
         if hunter is not None:
-            return _evaluate_hunter_pair(case, vuln_root, fix_root, hunter)
+            return _evaluate_hunter_pair(case, vuln_root, fix_root, hunter, _k_runs(k_runs))
         if case.slice in OVERLAY_SLICES:
             return _evaluate_overlay_pair(case, vuln_root, fix_root, ruleset)
         return _evaluate_inventory_pair(case, vuln_root, fix_root, ruleset=ruleset)
@@ -435,12 +441,15 @@ def evaluate_catalog(
     ruleset: tuple[PatternRule, ...] | None = None,
     inventory_sidecar: bool = True,
     pointers: bool | None = None,
+    k_runs: int | None = None,
 ) -> PairEvalResult:
     """Score every case. ``ruleset`` lets the improve loop score a candidate ledger; ``inventory_sidecar=False`` skips the second scan.
 
     Pointer pairs (``vendored = false``) are harvested into the cache when ``pointers`` (or OPENULTRASAST_PAIRS_NETWORK=1) allows the
     network, otherwise skipped with a ``pointer_pair_skipped`` degradation (Req 10.2, 10.3).
     """
+    if hunter is not None:
+        _k_runs(k_runs)  # refuse an impossible K before spending a single call
     degradations: list[dict[str, object]] = []
     ready: list[PairCase] = []
     for case in cases:
@@ -459,7 +468,7 @@ def evaluate_catalog(
         if inventory_sidecar:
             scorers[slice_name]["inventory"] = _metrics(tuple(_inventory_only(case, ruleset) for case in slice_cases))
         if hunter is not None:
-            hunted = tuple(evaluate_pair(case, hunter=hunter) for case in slice_cases)
+            hunted = tuple(evaluate_pair(case, hunter=hunter, k_runs=k_runs) for case in slice_cases)
             hunter_outcomes.extend(hunted)
             scorers[slice_name]["hunter"] = _metrics(hunted)
     if hunter is None and scorers:
@@ -938,10 +947,23 @@ def _with_functions(root: Path, findings: list[StaticFinding]) -> list[StaticFin
     return out
 
 
-def make_hunter_scan(client: object, model: str, *, max_steps: int | None = None) -> HunterScan:
-    """Run the tool hunter over every file of an isolated pair tree (Req 1.7)."""
+def make_hunter_scan(client: object, model: str, *, max_steps: int | None = None, taxonomy: object | None = None) -> HunterScan:
+    """Run the tool hunter over every file of an isolated pair tree (Req 1.7).
+
+    With a ``taxonomy`` the hunter is asked to name the family of every finding, from the closed list, and the answer
+    becomes the finding's `family:` tag. The class-aware scorer credits a detection only to a family, so a generalist
+    that names none can never be scored — it would report recall zero by construction rather than by measurement.
+    """
     from .complexity.map import Hotspot
-    from .tool_hunter import DEFAULT_MAX_STEPS, run_tool_hunter
+    from .tool_hunter import _SYSTEM_PROMPT, DEFAULT_MAX_STEPS, run_tool_hunter
+
+    prompt = _SYSTEM_PROMPT
+    if taxonomy is not None:
+        ids = ", ".join(family.id for family in taxonomy.families)  # type: ignore[attr-defined]
+        prompt = (
+            f'{_SYSTEM_PROMPT}\n\nGive every finding a "family" field, chosen from exactly this list: {ids}. '
+            'Use "unknown" when none of them fits; do not invent a name.'
+        )
 
     def scan(root: Path) -> list[StaticFinding]:
         hotspots = [
@@ -957,7 +979,17 @@ def make_hunter_scan(client: object, model: str, *, max_steps: int | None = None
             )
             for target in _targets(root)
         ]
-        return _with_functions(root, run_tool_hunter(root, hotspots, client=client, model=model, max_steps=max_steps or DEFAULT_MAX_STEPS))  # type: ignore[arg-type]
+        return _with_functions(
+            root,
+            run_tool_hunter(
+                root,
+                hotspots,
+                client=client,  # type: ignore[arg-type]
+                model=model,
+                max_steps=max_steps or DEFAULT_MAX_STEPS,
+                system_prompt=prompt,
+            ),
+        )
 
     return scan
 
@@ -1097,14 +1129,23 @@ def _evaluate_overlay_pair(case: PairCase, vuln_root: Path, fix_root: Path, rule
     )
 
 
-def _evaluate_hunter_pair(case: PairCase, vuln_root: Path, fix_root: Path, hunter: HunterScan, runs: int = 1) -> PairOutcome:
+def _k_runs(value: int | None) -> int:
+    """Req 3.5: three runs per pair per side is the floor, and the default, because one run misses most changes."""
+    from .learning.rounds import MIN_K_RUNS
+
+    runs = MIN_K_RUNS if value is None else int(value)
+    if runs < MIN_K_RUNS:
+        raise ValueError(f"the hunter path scores at least three runs per pair; {runs} was asked for")
+    return runs
+
+
+def _evaluate_hunter_pair(case: PairCase, vuln_root: Path, fix_root: Path, hunter: HunterScan, runs: int) -> PairOutcome:
     """Score the model-driven hunter by family and function (learning-harness Req 3.1-3.3, 3.5).
 
     Detection is a finding of the labeled family inside the labeled function; a finding of another family
     or outside it is counted and never called a leak; the words in a finding are never read. A pair that
     cannot be scored is reported with its reason and the detector is never run on it.
     """
-    from .learning.classify import classify_pair
     from .learning.families import load_families
     from .learning.scoring import score_pair_family, unscorable_reason
 
@@ -1115,10 +1156,9 @@ def _evaluate_hunter_pair(case: PairCase, vuln_root: Path, fix_root: Path, hunte
     if reason is not None:
         return _hunter_outcome(case, family=family, unscorable=reason)
     fix_ranges = _overlay_scan(fix_root).ranges
-    vuln_runs = [hunter(vuln_root) for _ in range(max(runs, 1))]
-    fix_runs = [hunter(fix_root) for _ in range(max(runs, 1))]
-    score = score_pair_family(case, family, vuln_runs, fix_runs, ranges=vuln_ranges, taxonomy=taxonomy)
-    del classify_pair, fix_ranges
+    vuln_runs = [hunter(vuln_root) for _ in range(runs)]
+    fix_runs = [hunter(fix_root) for _ in range(runs)]
+    score = score_pair_family(case, family, vuln_runs, fix_runs, ranges=vuln_ranges, taxonomy=taxonomy, fixed_ranges=fix_ranges)
     return _hunter_outcome(case, family=family, score=score, vuln=vuln_runs[0], fixed=fix_runs[0])
 
 

src/openultrasast/semantic/loo.py

@@ -99,11 +99,14 @@ def evaluate_loo(cases: Sequence[PairCase], *, facts: SemanticFacts | None = Non
             skipped.append((case.name, "pointer_pair_not_cached" if not case.vendored else "excerpt_missing"))
         else:
             targets.append(case)
-    from ..learning.split import is_teacher
+    from ..learning.split import is_teacher, refuse_if_holdout
 
     # learning-harness Req 5.1: every pair is still scored, but only a train-split teacher may seed a store.
     lessons = {case.name: pair_lessons(case, loaded) for case in targets if is_teacher(case)}
     outcomes = [_hold_out(case, targets, lessons, loaded) for case in targets]
+    # Req 5.2: the pairs that were kept out of the teaching set are named, not silently dropped.
+    refusal = refuse_if_holdout([case.name for case in targets if case.name not in lessons], targets)
+    refusals = (refusal.degradation(),) if refusal is not None else ()
     return LooResult(
         outcomes=tuple(outcomes),
         per_slice=_group(outcomes, lambda item: (item.slice,)),
@@ -112,7 +115,7 @@ def evaluate_loo(cases: Sequence[PairCase], *, facts: SemanticFacts | None = Non
         per_obligation_kind=_group(outcomes, lambda item: item.obligations),
         skipped=tuple(skipped),
         teaching_pairs=sum(1 for shapes in lessons.values() if shapes),
-        degradations=_unique_degradations(outcomes),
+        degradations=(*refusals, *_unique_degradations(outcomes)),
     )
 
 

src/openultrasast/semantic/seed.py

@@ -196,7 +196,9 @@ def _skip_reason(case: PairCase) -> str | None:
     from ..learning.split import is_teacher
 
     if not is_teacher(case) and case.review_tier in GATING_TIERS and case.vendored and not case.known_limit:
-        return f"split:{case.split}"  # learning-harness Req 5.1: only train-split, scorable pairs teach
+        # learning-harness Req 5.1: only train-split, scorable pairs teach. Say which of the two it was: two
+        # vibe-py rows are train-split identical twins, and calling them `split:train` is simply untrue.
+        return f"unscorable:{case.unscorable}" if case.unscorable else f"split:{case.split}"
     if case.review_tier not in GATING_TIERS:
         return f"tier:{case.review_tier}"
     if case.known_limit:

src/openultrasast/tool_hunter.py

@@ -175,8 +175,9 @@ class ChatClient(Protocol):
 class ScriptedChatClient:
     """Deterministic ChatClient for tests and OPENULTRASAST_HUNTER_CLIENT injection."""
 
-    def __init__(self, turns: list[ChatResponse]) -> None:
+    def __init__(self, turns: list[ChatResponse], *, cycle: bool = False) -> None:
         self._turns = turns
+        self._cycle = cycle  # a stub that runs out of answers turns every multi-run test into a silent zero
         self.calls: list[dict[str, object]] = []
 
     def complete(
@@ -192,7 +193,9 @@ class ScriptedChatClient:
         self.calls.append({"model": model, "messages": list(messages), "tools": tools, "timeout_seconds": timeout_seconds, **options})
         index = len(self.calls) - 1
         if index >= len(self._turns):
-            return ChatResponse(content="")
+            if not self._cycle or not self._turns:
+                return ChatResponse(content="")
+            index %= len(self._turns)
         return self._turns[index]
 
 
@@ -527,6 +530,10 @@ def _finding_from_item(
     severity = item.get("severity")
     snippet = item.get("snippet")
     proposed_snippet = snippet if isinstance(snippet, str) and snippet else None
+    family = item.get("family")
+    # The family is the only key the class-aware scorer credits, and it is the model's own answer: a fabricated
+    # name is counted as noise there rather than silently dropped here.
+    family_tags = [f"family:{family}"] if isinstance(family, str) and family.strip() else []
     return StaticFinding(
         finding_id=f"{_FINDING_ID_PREFIX}{relative}:{line_no if line_no is not None else 0}",
         path=relative,
@@ -540,7 +547,7 @@ def _finding_from_item(
         reachability_status="unknown",
         reachability_evidence=[],
         reachability_conditions=[],
-        tags=list(tags),
+        tags=[*tags, *family_tags],
         ranking_priority=float(scores.get(relative, 0.0)),
         proposed_snippet=proposed_snippet,
     )