fix(learning-harness): the round-3 review findings, remediated RED-first

Eight of nine group-3 tasks were rejected. Each defect was hiding a number,
and the numbers are why the loop was not learning:

- The fixed side was judged against the *vulnerable* side's function
  ranges. 191 of 227 pairs with a resolvable labeled span (84%) have
  different spans on the two sides, so a leak was recorded as silence,
  always in the direction that flatters Youden.
- Round zero measured every family with the generalist. `family:unknown`
  earns no credit against any real family, so every family but `unknown`
  baselined at a structural zero and every noise floor with it.
- The cross-family sweep counted absolute failures instead of negative
  flips. On a corpus where most pairs fail at round zero that rejects
  every round for damage it did not do. Unscorable rows voted too.
- K runs was declared and never threaded; `pairs --k-runs` was parsed and
  discarded, and its only test grepped `--help`.
- The meta-agent proposer was a stub, so `learning round` could never
  propose anything. It now composes MetaAgent.evolve against a scratch
  copy of the family directory, and `--proposals FILE` runs a recorded
  sequence offline.
- `refused_holdout` was a vocabulary word no code path could produce, and
  the covering test accepted either outcome.
- Cost was metered once, after everything had been paid for; a stage that
  raised left no record at all.
- The corrected lever number was published beside the earlier one rather
  than in place of it, and the test named for that property never checked.
- Refusals were `continue` statements: `Refusal.degradation()` existed and
  nothing called it. A train-split identical twin was reported as
  `split:train`, which is untrue.

Test repairs that each changed a number: two split tests measured an
`x = 1` fixture that teaches nothing whatever the rule says; two CLI
scoreboard tests asserted against all-zero output produced by a stub that
answers twice and puts its finding outside the labeled function.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

tests/test_learning_cli.py

@@ -19,6 +19,33 @@ CATALOG = (
 )
 
 
+def _detector(monkeypatch: pytest.MonkeyPatch, line: int = 6) -> None:
+    """A deterministic detector that reads the file it was given and answers differently on the two sides.
+
+    The shipped `scripted` stub answers twice and then goes quiet, and its finding sits at line 3, outside `run`.
+    Both make every family number zero, and a scoreboard assertion against zeros passes with the detector missing
+    entirely — which is what these tests used to assert."""
+    from openultrasast.learning.endpoint import ChatEndpoint
+    from openultrasast.tool_hunter import ChatResponse, ToolCall
+
+    class Detector:
+        def __init__(self) -> None:
+            self.turn = 0
+
+        def complete(self, *, model: str, messages, tools=None, **options):  # type: ignore[no-untyped-def]
+            del model, tools, options
+            self.turn += 1
+            if self.turn % 2 == 1:
+                return ChatResponse(tool_calls=(ToolCall(id=f"c{self.turn}", name="grep_repo", arguments={"pattern": "system"}),))
+            body = "\n".join(str(message.get("content") or "") for message in messages)
+            if "os.system" not in body:
+                return ChatResponse(content="[]")  # the fixed side really is different, and the detector sees it
+            return ChatResponse(content=json.dumps([{"path": "app.py", "line": line, "title": "t", "rationale": "r"}]))
+
+    endpoint = ChatEndpoint(provider="scripted", base_url="", thinking=False)
+    monkeypatch.setattr("openultrasast.learning.endpoint.resolve_chat_endpoint", lambda *a, **k: (Detector(), endpoint))
+
+
 @pytest.fixture
 def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
     for name in ("train-a", "hold-a"):
@@ -39,14 +66,25 @@ def test_classify_reports_the_corpus_and_writes_the_queue(workspace: Path, capsy
     assert "classified 2" in capsys.readouterr().out
 
 
-def test_score_reports_family_numbers_for_a_catalog(workspace: Path, capsys: pytest.CaptureFixture[str]) -> None:
+def test_score_reports_family_numbers_for_a_catalog(
+    workspace: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The scripted client detects the vulnerable side and is silent on the fix, so this is a real 2/2, not a zero.
+
+    An all-zero scoreboard would satisfy "the family key exists"; it is what a generalist detector scoring every
+    family produced before round zero ran each family's own configuration."""
+    _detector(monkeypatch)
     assert main(["learning", "score", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning"), "--json"]) == 0
     payload = json.loads(capsys.readouterr().out)
-    assert "injection" in payload["families"]
-    assert payload["families"]["injection"]["taxonomy_version"]
+    block = payload["families"]["injection"]
+    assert block["taxonomy_version"] and block["scorable"] == 2
+    assert block["outcomes"] == {"pair_correct": 2} and block["recall"] == 1.0 and block["youden"] == 1.0
 
 
-def test_baseline_writes_the_noise_floor_and_publish_reads_it(workspace: Path, capsys: pytest.CaptureFixture[str]) -> None:
+def test_baseline_writes_the_noise_floor_and_publish_reads_it(
+    workspace: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _detector(monkeypatch)
     assert (
         main(
             [
@@ -65,7 +103,12 @@ def test_baseline_writes_the_noise_floor_and_publish_reads_it(workspace: Path, c
         == 0
     )
     floors = json.loads((workspace / "learning" / "baseline" / "scripted" / "noise-floors.json").read_text())
-    assert floors["k_runs"] == 3 and "injection" in floors["floors"]
+    assert floors["k_runs"] == 3 and floors["floors"]["injection"]["pairs"] == 2
+    report = json.loads((workspace / "learning" / "baseline" / "scripted" / "report.json").read_text())
+    # Measured with the injection configuration, not the generalist: `family:unknown` earns no credit against any
+    # real family, so a baseline built from one shared detector reports a structural zero here.
+    assert report["metrics"]["injection"]["recall"] == 1.0
+    assert report["per_slice"]["vibe-py/injection"]["scorable"] == 2
     capsys.readouterr()
     roadmap = workspace / "roadmap.md"
     roadmap.write_text("# Roadmap\n\n## Overview\n\ntext\n")
@@ -116,10 +159,46 @@ def test_a_round_runs_from_the_command_line_and_journals_its_outcome(workspace:
     )
     rounds = [json.loads(line) for line in (workspace / "learning" / "journal.jsonl").read_text().splitlines() if line.strip()]
     assert len(rounds) == 1 and rounds[0]["family"] == "injection"
-    assert rounds[0]["outcome"] in {"accepted", "rejected", "reverted_cost"}
+    # Without the extra there is no meta-agent, so the honest outcome is exactly this one. Accepting any member of
+    # the enum would have hidden that the round could never propose anything at all.
+    assert (rounds[0]["outcome"], rounds[0]["reason"]) == ("rejected", "no_proposal")
     assert (workspace / "learning" / "rounds" / "1" / "attribution.json").is_file()
 
 
+def test_a_recorded_proposal_makes_the_round_do_something(workspace: Path) -> None:
+    """`--proposals` is what makes a round runnable at all without the optional extra."""
+    for command in (
+        ["learning", "baseline", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning"), "--k-runs", "3"],
+    ):
+        assert main([*command, "--model", "scripted"]) == 0
+    proposals = workspace / "proposals.json"
+    proposals.write_text(
+        json.dumps([{"hypothesis": "ask about quoting", "lever": "checklist", "change": {"checklist.md": "- ask about quoting\n"}}])
+    )
+    assert (
+        main(
+            [
+                "learning",
+                "round",
+                "--catalog",
+                str(workspace / "pairs.toml"),
+                "--out",
+                str(workspace / "learning"),
+                "--family",
+                "injection",
+                "--k-runs",
+                "3",
+                "--proposals",
+                str(proposals),
+            ]
+        )
+        == 0
+    )
+    (record,) = [json.loads(line) for line in (workspace / "learning" / "journal.jsonl").read_text().splitlines() if line.strip()]
+    assert record["reason"] != "no_proposal", "a recorded proposal reached the round"
+    assert json.loads((workspace / "learning" / "rounds" / "1" / "proposal.json").read_text())["lever"] == "checklist"
+
+
 def test_a_missing_endpoint_is_recorded_and_the_rest_still_runs(workspace: Path, monkeypatch: pytest.MonkeyPatch, capsys) -> None:  # type: ignore[no-untyped-def]
     for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"):
         monkeypatch.delenv(name, raising=False)
@@ -153,7 +232,24 @@ def test_every_subcommand_is_listed_in_help(capsys: pytest.CaptureFixture[str])
         assert name in out
 
 
-def test_the_pairs_command_takes_a_k_runs_flag(capsys: pytest.CaptureFixture[str]) -> None:
-    with pytest.raises(SystemExit):
-        main(["pairs", "--help"])
-    assert "--k-runs" in capsys.readouterr().out
+def test_the_pairs_command_scores_k_runs_per_side(workspace: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Grepping `--help` for the flag passes with the capability entirely absent, which is what it used to be."""
+    from openultrasast import pairs as pairs_module
+
+    calls: list[str] = []
+    real = pairs_module.make_hunter_scan
+
+    def counting(client: object, model: str, **kwargs: object):  # type: ignore[no-untyped-def]
+        inner = real(client, model, **kwargs)  # type: ignore[arg-type]
+
+        def scan(root: Path):  # type: ignore[no-untyped-def]
+            calls.append(root.name)
+            return inner(root)
+
+        return scan
+
+    monkeypatch.setattr(pairs_module, "make_hunter_scan", counting)
+    monkeypatch.setattr("openultrasast.cli.make_hunter_scan", counting)
+    monkeypatch.setenv("OPENULTRASAST_HUNTER_MODEL", "scripted")
+    assert main(["pairs", "--catalog", str(workspace / "pairs.toml"), "--hunter", "--hunter-model", "scripted", "--k-runs", "4"]) == 0
+    assert calls.count("vuln") == 8 and calls.count("fixed") == 8  # two pairs, four runs per side

tests/test_learning_pairs_evidence.py

@@ -0,0 +1,151 @@
+"""What the pairs hunter path must prove (learning-harness review round 3, Req 3.2, 3.5).
+
+Three defects, one test each: K was declared and never threaded, so every stage scored one run; the fixed
+side was judged against the *vulnerable* side's function ranges, which differ on most of the corpus; and
+the generalist hunter emitted findings with no family tag, which the class-aware scorer can never credit,
+so `ousast pairs --hunter` reported recall zero by construction.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from openultrasast.benchmark import ExpectedFinding
+from openultrasast.findings import StaticFinding
+from openultrasast.learning.families import load_families
+from openultrasast.pairs import PairCase, evaluate_catalog
+
+# The fix inserts two guard lines above the handler, so `run` starts on a different line on each side.
+VULN = "from flask import request\nimport os\n\n\ndef run():\n    os.system(request.args.get('cmd'))\n"
+FIXED = (
+    "from flask import request\nimport shlex\nimport os\n\n\n"
+    "def _safe(value):\n    return shlex.quote(value or '')\n\n\n"
+    "def run():\n    os.system(_safe(request.args.get('cmd')))\n"
+)
+
+
+def _case(tmp_path: Path, name: str = "p") -> PairCase:
+    (tmp_path / f"{name}-v.py").write_text(VULN)
+    (tmp_path / f"{name}-f.py").write_text(FIXED)
+    return PairCase(
+        name=name,
+        slice="vibe-py",
+        language="python",
+        origin="test",
+        vuln_file=tmp_path / f"{name}-v.py",
+        fixed_file=tmp_path / f"{name}-f.py",
+        relpath="app.py",
+        expected=(
+            ExpectedFinding(
+                cwe="CWE-78",
+                vulnerability_class="x",
+                path="app.py",
+                evidence="",
+                function="run",
+                mechanism="other",
+                family="injection",
+            ),
+        ),
+        min_recall=1.0,
+        fix_policy="silent",
+        review_tier="seeded",
+    )
+
+
+def _finding(line: int, *, family: str | None = "injection") -> StaticFinding:
+    return StaticFinding(
+        finding_id=f"tool-hunter:app.py:{line}",
+        path="app.py",
+        title="t",
+        severity="high",
+        confidence="low",
+        evidence_level="suspicion",
+        rationale="r",
+        line=line,
+        function_name="run",
+        reachability_status="unknown",
+        reachability_evidence=[],
+        reachability_conditions=[],
+        tags=["tool-hunter", *([f"family:{family}"] if family else [])],
+        ranking_priority=0.0,
+    )
+
+
+def test_the_k_runs_flag_reaches_the_detector(tmp_path: Path) -> None:
+    """Req 3.5: three runs per side, per pair. A parameter no caller can set is not a K-run floor."""
+    calls: list[str] = []
+
+    def scan(root: Path) -> list[StaticFinding]:
+        calls.append(root.name)
+        return [_finding(6)] if root.name == "vuln" else []
+
+    evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3)
+    assert calls.count("vuln") == 3 and calls.count("fixed") == 3
+
+
+def test_fewer_than_three_runs_is_refused(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match="three runs"):
+        evaluate_catalog([_case(tmp_path)], hunter=lambda root: [], k_runs=1)
+
+
+def test_the_fixed_side_is_judged_against_the_fixed_side_line_numbers(tmp_path: Path) -> None:
+    """`run` sits at line 5 on the vulnerable side and line 10 on the fixed side.
+
+    Scoring the fixed side against the vulnerable side's span reads a finding inside the fixed handler as
+    outside it, so a detector that flags both sides is recorded as silent on the fix — a leak counted as a
+    clean pass, and always in the direction that flatters the number."""
+
+    def scan(root: Path) -> list[StaticFinding]:
+        return [_finding(6)] if root.name == "vuln" else [_finding(11)]
+
+    result = evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3)
+    hunter = result.scorers["vibe-py"]["hunter"]
+    assert hunter.detected_vuln == 1
+    assert hunter.silent_fix == 0, "the finding is inside `run` on the fixed side; that is a leak"
+    assert hunter.pair_correct == 0
+
+
+def test_a_generalist_finding_without_a_family_is_never_a_detection(tmp_path: Path) -> None:
+    """The other half of the same defect: `pairs --hunter` used to feed the scorer untagged findings."""
+
+    def scan(root: Path) -> list[StaticFinding]:
+        return [_finding(6, family=None)] if root.name == "vuln" else []
+
+    result = evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3)
+    assert result.scorers["vibe-py"]["hunter"].detected_vuln == 0
+
+
+def test_the_hunter_reports_the_family_it_believes_and_that_is_what_is_credited(tmp_path: Path) -> None:
+    """A generalist that must name the family is answerable; one that names nothing cannot be scored at all."""
+    from openultrasast.tool_hunter import _findings_from_content
+
+    findings = _findings_from_content(
+        tmp_path,
+        [],
+        '[{"path": "app.py", "line": 6, "family": "injection", "title": "t", "rationale": "r"}]',
+    )
+    assert findings and "family:injection" in findings[0].tags
+    untagged = _findings_from_content(tmp_path, [], '[{"path": "app.py", "line": 6, "title": "t"}]')
+    assert untagged and not [tag for tag in untagged[0].tags if tag.startswith("family:")]
+
+
+def test_the_pair_hunter_asks_for_a_family_from_the_closed_list(tmp_path: Path) -> None:
+    """The prompt has to name the taxonomy, or the model invents family names and every finding is noise."""
+    from openultrasast.pairs import make_hunter_scan
+
+    seen: list[str] = []
+
+    class Client:
+        def complete(self, *, model: str, messages, tools=None, **options):  # type: ignore[no-untyped-def]
+            seen.append("\n".join(str(message.get("content") or "") for message in messages))
+            from openultrasast.tool_hunter import ChatResponse
+
+            return ChatResponse(content="[]", tool_calls=())
+
+    (tmp_path / "app.py").write_text(VULN)
+    make_hunter_scan(Client(), "m", taxonomy=load_families())(tmp_path)
+    prompt = "\n".join(seen)
+    assert "injection" in prompt and "access_control" in prompt and "untrusted_destination" in prompt
+    assert '"family"' in prompt

tests/test_learning_proposer.py

@@ -137,3 +137,86 @@ def test_the_harnessx_proposer_degrades_with_a_reason_when_the_extra_is_absent(t
     assert agent.propose(facts) is None
     assert agent.reason == "harnessx_unavailable"
     assert agent.write_roots("access_control") == (tmp_path / "configs" / "access_control",)
+
+
+# --- the meta-agent proposer (review round 3, task 3.5 finding 1) ------------
+
+
+def test_the_meta_agent_proposer_reads_its_change_out_of_a_scratch_copy(tmp_path: Path) -> None:
+    """The meta-agent never writes into the live family directory.
+
+    A proposer that edited the tree directly would have changed the configuration before the round snapshotted
+    it, so a rejection could not put it back. It gets a copy, and the proposal is the diff of that copy."""
+    from openultrasast.learning.proposer import HarnessXProposer
+
+    configs = tmp_path / "configs"
+    (configs / "injection").mkdir(parents=True)
+    (configs / "injection" / "checklist.md").write_text("- old\n")
+    (configs / "injection" / "family.toml").write_text('family = "injection"\nversion = "0"\n')
+    seen: dict[str, object] = {}
+
+    def agent(*, workspace: Path, facts, model: str):  # type: ignore[no-untyped-def]
+        seen["workspace"] = workspace
+        seen["family"] = facts.family
+        (workspace / "checklist.md").write_text("- ask about quoting\n")
+        return "ask about quoting"
+
+    proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent)
+    proposal = proposer.propose(_facts(tmp_path, "injection"))
+    assert proposal is not None
+    assert proposal.lever == "checklist" and proposal.change == {"checklist.md": "- ask about quoting\n"}
+    assert proposal.hypothesis == "ask about quoting"
+    assert seen["workspace"] != configs / "injection"
+    assert (configs / "injection" / "checklist.md").read_text() == "- old\n", "the live tree was edited"
+
+
+def test_the_meta_agent_proposer_refuses_more_than_one_changed_file(tmp_path: Path) -> None:
+    from openultrasast.learning.proposer import HarnessXProposer
+
+    configs = tmp_path / "configs"
+    (configs / "injection").mkdir(parents=True)
+    (configs / "injection" / "checklist.md").write_text("- old\n")
+    (configs / "injection" / "prompt.md").write_text("p\n")
+
+    def agent(*, workspace: Path, facts, model: str):  # type: ignore[no-untyped-def]
+        (workspace / "checklist.md").write_text("- a\n")
+        (workspace / "prompt.md").write_text("- b\n")
+        return "two at once"
+
+    proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent)
+    assert proposer.propose(_facts(tmp_path, "injection")) is None
+    assert "one file" in proposer.reason
+
+
+def test_the_meta_agent_proposer_refuses_a_file_no_lever_owns(tmp_path: Path) -> None:
+    from openultrasast.learning.proposer import HarnessXProposer
+
+    configs = tmp_path / "configs"
+    (configs / "injection").mkdir(parents=True)
+    (configs / "injection" / "checklist.md").write_text("- old\n")
+
+    def agent(*, workspace: Path, facts, model: str):  # type: ignore[no-untyped-def]
+        (workspace / "notes.txt").write_text("x\n")
+        return "sideways"
+
+    proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent)
+    assert proposer.propose(_facts(tmp_path, "injection")) is None
+    assert "notes.txt" in proposer.reason
+
+
+def test_the_meta_agent_proposer_says_so_when_it_changed_nothing(tmp_path: Path) -> None:
+    from openultrasast.learning.proposer import HarnessXProposer
+
+    configs = tmp_path / "configs"
+    (configs / "injection").mkdir(parents=True)
+    (configs / "injection" / "checklist.md").write_text("- old\n")
+    proposer = HarnessXProposer(configs_dir=configs, model="m", agent=lambda **_kwargs: "nothing to do")
+    assert proposer.propose(_facts(tmp_path, "injection")) is None
+    assert proposer.reason == "no_change_proposed"
+
+
+def _facts(tmp_path: Path, family: str):  # type: ignore[no-untyped-def]
+    from openultrasast.learning.proposer import FailureFacts
+
+    del tmp_path
+    return FailureFacts(family=family, config=_config(family))

tests/test_learning_publish.py

@@ -107,13 +107,22 @@ def test_the_corrected_lever_number_replaces_the_earlier_one(tmp_path: Path) ->
         )
     )
     roadmap = tmp_path / "roadmap.md"
-    roadmap.write_text(ROADMAP + "\nThe lever recovered 7 of 17 holdout pairs at Youden +0.059.\n")
+    stale = "human holdout profile pair_correct 0 -> 6 of 17, Youden 0 -> +0.353, round 2 `no_proposals`."
+    roadmap.write_text(f"{ROADMAP}\nMeasured 2026-09-05 with the vibe-py candidates: {stale}\n")
     report = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
     text = roadmap.read_text()
     assert "-0.0588" in report.table or "-0.059" in report.table
     assert "5 of 17" in report.table or "detected 5" in report.table
     assert "holdout pairs stopped teaching" in report.table.lower() or "train split only" in report.table.lower()
     assert text.count("learning-harness:begin") == 1
+    # Req 5.3 says *in place of* the earlier one. Leaving the old claim above the new section is how a document
+    # ends up asserting two different numbers for the same measurement.
+    assert stale not in text
+    assert "superseded" in text and "learning-harness" in text
+    assert "Measured 2026-09-05 with the vibe-py candidates:" in text, "the sentence is corrected, not deleted"
+    before = roadmap.read_text()
+    publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
+    assert roadmap.read_text() == before, "a second publish must be a no-op once the claim is corrected"
 
 
 def test_the_roadmap_section_is_replaced_not_appended_again(tmp_path: Path) -> None:

tests/test_learning_refusals.py

@@ -0,0 +1,94 @@
+"""A refusal is data, not a `continue` (learning-harness review round 3, Req 5.1, 5.2).
+
+`Refusal.degradation()` existed and nothing ever called it, so every path that declined to learn from a pair
+declined silently: the mechanism lever skipped a holdout-taught shape without a word, leave-one-out dropped
+non-teachers with no record, and the seeder gave a train-split identical twin the reason `split:train`, which
+is not true. A leak the run does not mention is a leak nobody reads.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from openultrasast.benchmark import ExpectedFinding
+from openultrasast.pairs import PairCase
+
+VULN = "import os\nfrom flask import request\n\n\ndef run():\n    os.system(request.args.get('cmd'))\n"
+FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n    subprocess.run(['echo'], check=False)\n"
+
+
+def _case(tmp_path: Path, name: str, *, split: str = "train", unscorable: str | None = None, tier: str = "seeded") -> PairCase:
+    (tmp_path / f"{name}-v.py").write_text(VULN)
+    (tmp_path / f"{name}-f.py").write_text(FIXED)
+    return PairCase(
+        name=name,
+        slice="vibe-py",
+        language="python",
+        origin="test",
+        vuln_file=tmp_path / f"{name}-v.py",
+        fixed_file=tmp_path / f"{name}-f.py",
+        relpath="app.py",
+        expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),),
+        min_recall=1.0,
+        fix_policy="silent",
+        split=split,
+        review_tier=tier,
+        unscorable=unscorable,
+    )
+
+
+def test_a_train_split_twin_is_refused_for_being_a_twin_not_for_its_split(tmp_path: Path) -> None:
+    """`split:train` on a train-split row is a lie, and it hides a corpus fact behind a split fact.
+
+    Two vibe-py rows are exactly this: `python-insecure-app-main-try-hack-me` and
+    `vulnpy-deserialization-do-pickle-load` are train-split identical twins."""
+    from openultrasast.semantic.seed import _skip_reason
+
+    assert _skip_reason(_case(tmp_path, "twin", split="train", unscorable="identical_twin")) == "unscorable:identical_twin"
+    assert _skip_reason(_case(tmp_path, "held", split="holdout")) == "split:holdout"
+    assert _skip_reason(_case(tmp_path, "good", split="train")) is None
+
+
+def test_leave_one_out_records_the_pairs_it_would_not_learn_from(tmp_path: Path) -> None:
+    """Req 5.2: the refusal is recorded with the pair names, on a channel a caller can read."""
+    from openultrasast.semantic.loo import evaluate_loo
+
+    result = evaluate_loo([_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")])
+    refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"]
+    assert refusals, "leave-one-out excluded a holdout pair from teaching and said nothing"
+    assert refusals[0]["pairs"] == ["hold-a"]
+    assert result.teaching_pairs <= 1
+
+
+def test_the_mechanism_lever_records_every_shape_a_holdout_pair_taught(tmp_path: Path) -> None:
+    """The lever used to `continue` past a refused candidate; the run then showed a shorter list with no reason."""
+    from openultrasast.improve.evolve import propose_mechanism_edits
+    from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
+    from openultrasast.semantic.variants import Shape
+
+    cases = [_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")]
+    candidates = MechanismStore(tmp_path / "candidates.jsonl")
+    append_from_pair(
+        candidates,
+        Shape(
+            language="python",
+            sink_name="system",
+            arity=1,
+            source_positions=(0,),
+            source_kinds=("request",),
+            guard="none",
+            mechanism="source_reaches_sink",
+        ),
+        summary="s",
+        cwe="CWE-78",
+        pair="hold-a",
+        provenance="human",
+        tier="seeded",
+    )
+    refusals: list[dict[str, object]] = []
+    edits = propose_mechanism_edits(cases, candidates, tmp_path / "scan.jsonl", degradations=refusals)
+    assert [edit.mechanism_id for edit in edits if edit.action == "admit"] == []
+    assert [item["reason"] for item in refusals] == ["holdout_pair_refused"]
+    assert refusals[0]["pairs"] == ["hold-a"]
+    json.dumps(refusals)  # a degradation has to survive the artifact it is written into

tests/test_learning_round_evidence.py

@@ -0,0 +1,375 @@
+"""What a round must prove before it is kept (learning-harness review round 3, Req 4.1, 5.2, 9.5, 9.6, 9.8).
+
+The round loop was accepting and rejecting on numbers that did not mean what their names said: the
+cross-family sweep counted every failing pair rather than the pairs this change broke, unscorable rows
+voted against a round they cannot judge, cost was metered once after everything had been paid for, and a
+stage that raised left no record at all. Each test here fails on exactly one of those.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from openultrasast.benchmark import ExpectedFinding
+from openultrasast.findings import StaticFinding
+from openultrasast.learning.detectors import FamilyConfig, write_default_configs
+from openultrasast.learning.families import load_families
+from openultrasast.learning.journal import Archive, LearningJournal
+from openultrasast.learning.proposer import Proposal, ScriptedProposer
+from openultrasast.learning.rounds import NoiseFloor, run_learning_round
+from openultrasast.pairs import PairCase
+
+VULN = "from flask import request\nimport os\n\n\ndef run():\n    os.system(request.args.get('cmd'))\n"
+FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n    subprocess.run(['echo', 'x'], check=False)\n"
+
+
+def _case(tmp_path: Path, name: str, *, split: str, family: str = "injection", unscorable: str | None = None) -> PairCase:
+    (tmp_path / f"{name}-v.py").write_text(VULN)
+    (tmp_path / f"{name}-f.py").write_text(FIXED)
+    return PairCase(
+        name=name,
+        slice="vibe-py",
+        language="python",
+        origin="test",
+        vuln_file=tmp_path / f"{name}-v.py",
+        fixed_file=tmp_path / f"{name}-f.py",
+        relpath="app.py",
+        expected=(
+            ExpectedFinding(
+                cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family
+            ),
+        ),
+        min_recall=1.0,
+        fix_policy="silent",
+        split=split,
+        review_tier="seeded",
+        unscorable=unscorable,
+    )
+
+
+def _finding(family: str) -> StaticFinding:
+    return StaticFinding(
+        finding_id="detector:app.py:6",
+        path="app.py",
+        title="t",
+        severity="high",
+        confidence="low",
+        evidence_level="suspicion",
+        rationale="r",
+        line=6,
+        function_name="run",
+        reachability_status="unknown",
+        reachability_evidence=[],
+        reachability_conditions=[],
+        tags=[f"family:{family}", "detector:x@0"],
+        ranking_priority=0.0,
+    )
+
+
+def _proposal(text: str = "- ask about quoting\n") -> Proposal:
+    return Proposal(hypothesis="ask about quoting", lever="checklist", change={"checklist.md": text}, predicted_affected=("train-a",))
+
+
+@pytest.fixture
+def world(tmp_path: Path):  # type: ignore[no-untyped-def]
+    configs = tmp_path / "configs"
+    write_default_configs(configs, load_families(), prompt="p", version="0")
+    (configs / "injection" / "checklist.md").write_text("- old\n")
+    return {
+        "configs": configs,
+        "cases": [_case(tmp_path, "train-a", split="train"), _case(tmp_path, "hold-a", split="holdout")],
+        "journal": LearningJournal(tmp_path / "journal.jsonl"),
+        "archive": Archive(tmp_path / "archive.jsonl"),
+        "out": tmp_path / "out",
+        "floors": {"injection": NoiseFloor(family="injection", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0)},
+        "root": tmp_path,
+    }
+
+
+def _run(world, proposal, build, **kwargs):  # type: ignore[no-untyped-def]
+    return run_learning_round(
+        world["cases"],
+        family="injection",
+        taxonomy=load_families(),
+        configs_dir=world["configs"],
+        proposer=ScriptedProposer([proposal] if proposal else []),
+        scan_factory=build,
+        model="m",
+        journal=world["journal"],
+        archive=world["archive"],
+        floors=world["floors"],
+        out_dir=world["out"],
+        **kwargs,
+    )
+
+
+def _injection_improves(config: FamilyConfig):  # type: ignore[no-untyped-def]
+    """Injection detects only once the proposed checklist lands; every other family finds nothing, ever."""
+
+    def scan(root: Path) -> list[StaticFinding]:
+        if root.name != "vuln" or config.family != "injection":
+            return []
+        return [_finding("injection")] if config.checklist.strip() == "- ask about quoting" else []
+
+    return scan
+
+
+# --- the sweep ---------------------------------------------------------------
+
+
+def test_a_family_that_was_already_failing_does_not_reject_the_round(world) -> None:  # type: ignore[no-untyped-def]
+    """Req 9.5 counts *negative flips*, not absolute failures.
+
+    On the real corpus most pairs fail at round zero, so counting failures rejects every round for damage it
+    did not do — and the harness then reports that it cannot improve."""
+    world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
+    world["floors"]["access_control"] = NoiseFloor(
+        family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
+    )
+    record = _run(world, _proposal(), _injection_improves, sweep_families=("access_control",))
+    assert record.sweep["access_control"] == 0, "access control failed before and after; this change flipped nothing"
+    assert record.outcome == "accepted" and record.reason == "improved"
+
+
+def test_a_family_this_change_really_broke_still_rejects_the_round(world) -> None:  # type: ignore[no-untyped-def]
+    """The other arm: a pair that was correct before and is not after is a flip, and one is over the budget."""
+    world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
+    world["floors"]["access_control"] = NoiseFloor(
+        family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
+    )
+    landed = {"yes": False}
+
+    def build(config: FamilyConfig):  # type: ignore[no-untyped-def]
+        def scan(root: Path) -> list[StaticFinding]:
+            if root.name != "vuln":
+                return []
+            if config.family == "access_control":
+                return [] if landed["yes"] else [_finding("access_control")]
+            if config.checklist.strip() == "- ask about quoting":
+                landed["yes"] = True
+                return [_finding("injection")]
+            return []
+
+        return scan
+
+    record = _run(world, _proposal(), build, sweep_families=("access_control",))
+    assert record.sweep["access_control"] == 1
+    assert record.outcome == "rejected" and record.reason == "collateral_regression"
+    assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
+
+
+def test_an_unscorable_row_never_votes_in_the_sweep(world) -> None:  # type: ignore[no-untyped-def]
+    """Req 4.1: an unscorable row leaves every denominator, and it certainly does not reject a round."""
+    world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control", unscorable="identical_twin"))
+    world["floors"]["access_control"] = NoiseFloor(
+        family="access_control", k_runs=5, pairs=0, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
+    )
+    record = _run(world, _proposal(), _injection_improves, sweep_families=("access_control",))
+    assert record.sweep["access_control"] == 0 and record.outcome == "accepted"
+
+
+def test_attribution_names_the_pairs_this_change_flipped_and_no_others(world) -> None:  # type: ignore[no-untyped-def]
+    world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
+    world["floors"]["access_control"] = NoiseFloor(
+        family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
+    )
+    _run(world, _proposal(), _injection_improves, sweep_families=("access_control",))
+    attribution = json.loads((world["out"] / "rounds" / "1" / "attribution.json").read_text())
+    assert attribution["flipped_predicted"] == ["hold-a", "train-a"]
+    assert attribution["flipped_unpredicted"] == [], "a pair that was already failing was never flipped by this round"
+
+
+# --- cost, exceptions, the archive ------------------------------------------
+
+
+def test_the_cap_stops_a_round_before_it_pays_for_the_next_stage(world) -> None:  # type: ignore[no-untyped-def]
+    """Req 9.8 says stop, revert and record. Metering once, after every stage, can only ever record."""
+    stages: list[str] = []
+
+    def build(config: FamilyConfig):  # type: ignore[no-untyped-def]
+        def scan(root: Path) -> list[StaticFinding]:
+            stages.append(f"{config.family}:{root.name}")
+            return []
+
+        return scan
+
+    spend = {"usd": 0.0}
+
+    def spent() -> float:
+        spend["usd"] += 1.0
+        return spend["usd"]
+
+    world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
+    record = _run(world, _proposal(), build, cost_cap_usd=0.5, spent_usd=spent, sweep_families=("access_control",))
+    assert record.outcome == "reverted_cost" and record.reason == "cost_cap_exceeded"
+    assert "access_control:vuln" not in stages, "the sweep ran after the cap was already crossed"
+    assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
+
+
+def test_a_stage_that_raises_is_reverted_journalled_and_re_raised(world) -> None:  # type: ignore[no-untyped-def]
+    """An endpoint failure mid-round is an outcome. Losing the round record loses the money it already spent."""
+
+    def build(config: FamilyConfig):  # type: ignore[no-untyped-def]
+        def scan(root: Path) -> list[StaticFinding]:
+            if config.checklist.strip() == "- ask about quoting":
+                raise RuntimeError("endpoint said no")
+            return []
+
+        return scan
+
+    with pytest.raises(RuntimeError, match="endpoint said no"):
+        _run(world, _proposal(), build)
+    assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
+    outcomes = [(item.outcome, item.reason) for item in world["journal"].rounds()]
+    assert outcomes == [("reverted", "stage_failed")]
+    assert (world["out"] / "rounds" / "1" / "scores.json").is_file()
+
+
+def test_a_rejected_version_is_archived_so_a_pair_it_alone_wins_is_not_lost(world) -> None:  # type: ignore[no-untyped-def]
+    """Req 9.7: per-pair winners, or the loop collapses onto one lineage and forgets what a rejected try could do.
+
+    This round wins both of its own pairs and is still rejected, because it broke another family. Version 1 is the
+    only configuration that ever got those two pairs right; dropping it because the round as a whole lost is how a
+    search collapses onto one lineage."""
+    world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control"))
+    world["floors"]["access_control"] = NoiseFloor(
+        family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
+    )
+    landed = {"yes": False}
+
+    def build(config: FamilyConfig):  # type: ignore[no-untyped-def]
+        def scan(root: Path) -> list[StaticFinding]:
+            if root.name != "vuln":
+                return []
+            if config.family == "access_control":
+                return [] if landed["yes"] else [_finding("access_control")]
+            if config.checklist.strip() == "- ask about quoting":
+                landed["yes"] = True
+                return [_finding("injection")]
+            return []
+
+        return scan
+
+    record = _run(world, _proposal(), build, sweep_families=("access_control",))
+    assert record.outcome == "rejected" and record.reason == "collateral_regression"
+    assert world["archive"].winners("injection") == {"train-a": "1", "hold-a": "1"}
+    assert 'version = "0"' in (world["configs"] / "injection" / "family.toml").read_text(), "a rejected round is not versioned"
+
+
+# --- the holdout refusal -----------------------------------------------------
+
+
+def test_a_proposal_that_names_a_holdout_pair_is_refused_by_name(world) -> None:  # type: ignore[no-untyped-def]
+    """Req 5.2: a candidate that touches a holdout pair is refused and the refusal names the pairs."""
+    record = _run(
+        world,
+        Proposal(
+            hypothesis="memorize the answer",
+            lever="memory",
+            change={"hard_negatives.jsonl": json.dumps({"pair": "hold-a", "note": "always flag"}) + "\n"},
+            predicted_affected=("hold-a",),
+        ),
+        _injection_improves,
+    )
+    assert record.outcome == "refused_holdout"
+    assert "hold-a" in record.reason
+    assert (world["configs"] / "injection" / "hard_negatives.jsonl").read_text() == ""
+
+
+def test_a_proposal_that_only_names_train_pairs_is_not_refused(world) -> None:  # type: ignore[no-untyped-def]
+    record = _run(
+        world,
+        Proposal(
+            hypothesis="remember this shape",
+            lever="memory",
+            change={"hard_negatives.jsonl": json.dumps({"pair": "train-a", "note": "shape"}) + "\n"},
+            predicted_affected=("train-a",),
+        ),
+        _injection_improves,
+    )
+    assert record.outcome != "refused_holdout"
+
+
+# --- round zero --------------------------------------------------------------
+
+
+def test_round_zero_measures_each_family_with_its_own_detector(tmp_path: Path) -> None:
+    """Req 8.2. One detector for every family means the generalist scores them all, and abstention earns no credit
+    (`related(X, "unknown")` is lateral), so every family but `unknown` baselines at a structural zero."""
+    from openultrasast.learning.rounds import run_baseline
+
+    configs = tmp_path / "configs"
+    write_default_configs(configs, load_families(), prompt="p", version="0")
+    seen: list[str] = []
+
+    def factory(config: FamilyConfig):  # type: ignore[no-untyped-def]
+        def scan(root: Path) -> list[StaticFinding]:
+            seen.append(config.family)
+            return [_finding(config.family)] if root.name == "vuln" else []
+
+        return scan
+
+    cases = [_case(tmp_path, "inj", split="train"), _case(tmp_path, "acl", split="train", family="access_control")]
+    report = run_baseline(
+        cases,
+        taxonomy=load_families(),
+        configs_dir=configs,
+        scan_factory=factory,
+        model="m",
+        out_dir=tmp_path / "out",
+        k_runs=3,
+    )
+    assert set(seen) == {"injection", "access_control"}, "each family must be measured with its own configuration"
+    assert report.metrics["injection"]["recall"] == 1.0 and report.metrics["access_control"]["recall"] == 1.0
+
+
+def test_round_zero_reports_each_slice_separately(tmp_path: Path) -> None:
+    """Req 8.4: vibe-py and agent-vfc are different worlds; one pooled number hides which one moved."""
+    from openultrasast.learning.rounds import run_baseline
+
+    configs = tmp_path / "configs"
+    write_default_configs(configs, load_families(), prompt="p", version="0")
+    web = _case(tmp_path, "web", split="train")
+    agent = _case(tmp_path, "agent", split="train")
+    object.__setattr__(agent, "slice", "agent-vfc")
+
+    def factory(config: FamilyConfig):  # type: ignore[no-untyped-def]
+        def scan(root: Path) -> list[StaticFinding]:
+            return [_finding(config.family)] if root.name == "vuln" else []
+
+        return scan
+
+    report = run_baseline(
+        [web, agent],
+        taxonomy=load_families(),
+        configs_dir=configs,
+        scan_factory=factory,
+        model="m",
+        out_dir=tmp_path / "out",
+        k_runs=3,
+    )
+    assert set(report.per_slice) == {"vibe-py/injection", "agent-vfc/injection"}
+    payload = json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text())
+    assert set(payload["per_slice"]) == {"vibe-py/injection", "agent-vfc/injection"}
+
+
+def test_round_zero_refuses_fewer_than_three_runs(tmp_path: Path) -> None:
+    """The floor exists because a single run misses most real changes; a baseline measured at K=1 is a noise floor
+    of zero and every later round then inherits a budget of nothing."""
+    from openultrasast.learning.rounds import run_baseline
+
+    configs = tmp_path / "configs"
+    write_default_configs(configs, load_families(), prompt="p", version="0")
+    with pytest.raises(ValueError, match="three runs"):
+        run_baseline(
+            [_case(tmp_path, "inj", split="train")],
+            taxonomy=load_families(),
+            configs_dir=configs,
+            scan_factory=lambda config: lambda root: [],
+            model="m",
+            out_dir=tmp_path / "out",
+            k_runs=1,
+        )

tests/test_learning_round_loop.py

@@ -134,21 +134,29 @@ def test_a_change_that_helps_nothing_is_rejected_and_reverted(world) -> None:  #
 
 
 def test_a_round_is_rejected_when_the_sweep_finds_another_family_below_its_floor(world) -> None:  # type: ignore[no-untyped-def]
-    """The sweep is a floor check over every other family, not a causal claim about this change.
+    """The sweep counts pairs this change turned from right to wrong in another family (Req 9.5).
 
-    Per-family directories make a change that reaches another family structurally impossible, so what the
-    sweep really guards is a family that is failing for any reason at the moment this round would land.
+    It used to count every failing pair, which rejected a round for damage it did not do; the pair below is right
+    before the change and wrong after it, which is the only thing that may reject a round on another family's
+    behalf. The other arm — a family that was already failing must not reject — lives in
+    `tests/test_learning_round_evidence.py`.
     """
     world["cases"].append(_case(world["configs"].parent, "other-hold", split="holdout", family="access_control"))
     world["floors"]["access_control"] = NoiseFloor(
         family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
     )
+    landed = {"yes": False}
 
     def build(config: FamilyConfig):  # type: ignore[no-untyped-def]
         def scan(root: Path) -> list[StaticFinding]:
-            if root.name != "vuln" or config.family == "access_control":
-                return []  # access control finds nothing, so its holdout pair is below its floor
-            return [_finding("injection")] if config.checklist.strip() == "- ask about quoting" else []
+            if root.name != "vuln":
+                return []
+            if config.family == "access_control":
+                return [] if landed["yes"] else [_finding("access_control")]
+            if config.checklist.strip() == "- ask about quoting":
+                landed["yes"] = True  # the injection change lands, and access control stops finding its pair
+                return [_finding("injection")]
+            return []
 
         return scan
 
@@ -181,8 +189,7 @@ def test_a_round_over_its_cost_cap_is_reverted_and_says_so(world) -> None:  # ty
 
 def test_a_proposal_that_reaches_beyond_its_family_never_runs(world) -> None:  # type: ignore[no-untyped-def]
     record = _run(world, Proposal(hypothesis="h", lever="checklist", change={"../escape.md": "x"}), {})
-    assert record.outcome == "refused_holdout" or record.outcome == "rejected"
-    assert "outside" in record.reason
+    assert record.outcome == "rejected" and "outside" in record.reason  # the holdout refusal has its own test
     assert not (world["configs"].parent / "escape.md").exists()
 
 
@@ -202,7 +209,7 @@ def test_the_round_directory_holds_what_a_replay_needs(world) -> None:  # type:
     directory = world["out"] / "rounds" / "1"
     assert (directory / "proposal.json").is_file() and (directory / "scores.json").is_file()
     attribution = json.loads((directory / "attribution.json").read_text())
-    assert attribution["flipped_predicted"] == ["hold-a"] or attribution["flipped_predicted"] == ["train-a", "hold-a"]
+    assert attribution["flipped_predicted"] == ["hold-a", "train-a"]  # the round is deterministic; one exact answer
     assert attribution["flipped_unpredicted"] == []
     trajectories = (directory / "trajectories.jsonl").read_text()
     assert "REDACTED" in trajectories and "B" * 24 not in trajectories

tests/test_learning_rounds.py

@@ -70,6 +70,16 @@ def _steady(root: Path) -> list[StaticFinding]:
     return [_finding("injection")] if _is_vulnerable_side(root) else []
 
 
+def _always(scan):  # type: ignore[no-untyped-def]
+    """Round zero asks for a detector per family; these tests deliberately give every family the same one so the
+    noise-floor arithmetic is the only thing under test."""
+
+    def build(_config):  # type: ignore[no-untyped-def]
+        return scan
+
+    return build
+
+
 def _flaky(counter: list[int]):  # type: ignore[no-untyped-def]
     def scan(root: Path) -> list[StaticFinding]:
         if not _is_vulnerable_side(root):
@@ -86,7 +96,12 @@ def test_round_zero_writes_a_configuration_for_every_family(tmp_path: Path) -> N
 
     taxonomy = load_families()
     report = run_baseline(
-        [_case(tmp_path, "a")], taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan=_steady, model="m", out_dir=tmp_path / "out"
+        [_case(tmp_path, "a")],
+        taxonomy=taxonomy,
+        configs_dir=tmp_path / "configs",
+        scan_factory=_always(_steady),
+        model="m",
+        out_dir=tmp_path / "out",
     )
     configs = load_family_configs(tmp_path / "configs", taxonomy)
     assert set(configs) == {family.id for family in taxonomy.families}
@@ -102,7 +117,7 @@ def test_a_steady_detector_has_a_zero_noise_floor_and_a_flaky_one_does_not(tmp_p
         [_case(tmp_path, "a"), _case(tmp_path, "b")],
         taxonomy=taxonomy,
         configs_dir=tmp_path / "c1",
-        scan=_steady,
+        scan_factory=_always(_steady),
         model="m",
         out_dir=tmp_path / "o1",
     )
@@ -112,7 +127,7 @@ def test_a_steady_detector_has_a_zero_noise_floor_and_a_flaky_one_does_not(tmp_p
         [_case(tmp_path, "a"), _case(tmp_path, "b")],
         taxonomy=taxonomy,
         configs_dir=tmp_path / "c2",
-        scan=_flaky([]),
+        scan_factory=_always(_flaky([])),
         model="m",
         out_dir=tmp_path / "o2",
     )
@@ -127,7 +142,7 @@ def test_the_memory_family_is_measured_but_never_gates(tmp_path: Path) -> None:
         [_case(tmp_path, "m", family="memory", slice_name="vfc")],
         taxonomy=taxonomy,
         configs_dir=tmp_path / "configs",
-        scan=_steady,
+        scan_factory=_always(_steady),
         model="m",
         out_dir=tmp_path / "out",
         slices=("vfc",),
@@ -141,7 +156,9 @@ def test_the_default_selection_is_the_web_slices(tmp_path: Path) -> None:
 
     taxonomy = load_families()
     cases = [_case(tmp_path, "web", slice_name="vibe-py"), _case(tmp_path, "c", family="memory", slice_name="vfc")]
-    report = run_baseline(cases, taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan=_steady, model="m", out_dir=tmp_path / "out")
+    report = run_baseline(
+        cases, taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan_factory=_always(_steady), model="m", out_dir=tmp_path / "out"
+    )
     assert set(report.metrics) == {"injection"} and "memory" not in report.floors
 
 
@@ -155,7 +172,7 @@ def test_artifacts_are_keyed_by_model_so_another_model_can_be_compared(tmp_path:
             [_case(tmp_path, f"a{index}")],
             taxonomy=taxonomy,
             configs_dir=tmp_path / "configs",
-            scan=_steady,
+            scan_factory=_always(_steady),
             model=model,
             out_dir=out,
         )
@@ -184,7 +201,9 @@ def test_an_unscorable_pair_is_listed_and_never_run(tmp_path: Path) -> None:
     taxonomy = load_families()
     case = _case(tmp_path, "twin")
     twin = PairCase(**{**case.__dict__, "unscorable": "identical_twin"})
-    report = run_baseline([twin], taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan=counting, model="m", out_dir=tmp_path / "out")
+    report = run_baseline(
+        [twin], taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan_factory=_always(counting), model="m", out_dir=tmp_path / "out"
+    )
     assert report.metrics["injection"]["scorable"] == 0
     assert report.metrics["injection"]["unscorable"] == {"identical_twin": 1}
     assert calls == []
@@ -198,7 +217,7 @@ def test_round_zero_records_what_it_spent(tmp_path: Path) -> None:
         [_case(tmp_path, "a")],
         taxonomy=load_families(),
         configs_dir=tmp_path / "configs",
-        scan=_steady,
+        scan_factory=_always(_steady),
         model="m",
         out_dir=tmp_path / "out",
         spent_usd=lambda: 1.25,

tests/test_learning_scan.py

@@ -91,9 +91,13 @@ def test_family_findings_never_become_sandbox_candidates(tmp_path: Path, monkeyp
     assert main(["scan", str(repo), "--mode", "deep", "--config", str(repo / "openultrasast.toml")]) == 0
     run_dir = _run_dir(repo)
     verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"]
-    findings = {item["finding_id"] for item in json.loads((run_dir / "findings.json").read_text())["findings"]}
-    del findings
-    assert all(not str(item.get("path", "")).startswith("family:") for item in verdicts)
+    payload = json.loads((run_dir / "findings.json").read_text())["findings"]
+    family_findings = {item["finding_id"] for item in payload if any(str(tag).startswith("family:") for tag in item["tags"])}
+    assert family_findings, "the fixture must actually produce family findings, or this proves nothing"
+    # A family detector's claim is a suspicion until a verifier speaks; handing it to the sandbox would let an
+    # unverified model claim drive execution (Req 7.1).
+    candidates = {str(item.get("finding_id") or item.get("id") or "") for item in verdicts}
+    assert candidates.isdisjoint(family_findings)
 
 
 def test_the_sarif_report_carries_the_family_properties(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:

tests/test_learning_split.py

@@ -8,6 +8,33 @@ from pathlib import Path
 from openultrasast.benchmark import ExpectedFinding
 from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_slice, select_vendored
 
+VULN = "import os\nfrom flask import request\n\n\ndef run():\n    os.system(request.args.get('cmd'))\n"
+FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n    subprocess.run(['echo'], check=False)\n"
+
+
+def _teaching_case(tmp_path: Path, name: str, *, split: str = "train") -> PairCase:
+    """A pair that really carries a mechanism shape.
+
+    The trivial `x = 1` fixture below teaches nothing whatever the split says, so a teacher-rule assertion made
+    against it passes with the rule deleted. This one is the difference between measuring the rule and measuring
+    an empty corpus."""
+    (tmp_path / f"{name}-v.py").write_text(VULN)
+    (tmp_path / f"{name}-f.py").write_text(FIXED)
+    return PairCase(
+        name=name,
+        slice="vibe-py",
+        language="python",
+        origin="test",
+        vuln_file=tmp_path / f"{name}-v.py",
+        fixed_file=tmp_path / f"{name}-f.py",
+        relpath="app.py",
+        expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),),
+        min_recall=1.0,
+        fix_policy="silent",
+        split=split,
+        review_tier="seeded",
+    )
+
 
 def _case(tmp_path: Path, name: str, *, split: str = "train", tier: str = "seeded", unscorable: str | None = None) -> PairCase:
     (tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n")
@@ -69,33 +96,45 @@ def test_the_exporter_seeds_from_train_pairs_only(tmp_path: Path) -> None:
 
 
 def test_leave_one_out_teaches_from_train_pairs_only(tmp_path: Path) -> None:
+    """Both arms, because `teaching_pairs <= 1` on a two-pair corpus is satisfied by a rule that does nothing.
+
+    The same two excerpts teach twice when both are on the train split and once when one is held out, and the run
+    names the pair it refused (Req 5.1, 5.2)."""
     from openultrasast.semantic.loo import evaluate_loo
 
-    cases = [_case(tmp_path, "train-one"), _case(tmp_path, "holdout-one", split="holdout")]
-    result = evaluate_loo(cases)
+    both_train = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "train-two")])
+    assert both_train.teaching_pairs == 2 and both_train.degradations == ()
+    result = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")])
     assert {outcome.pair for outcome in result.outcomes} == {"train-one", "holdout-one"}  # every pair is still scored
-    assert result.teaching_pairs <= 1  # only the train pair could ever have taught
+    assert result.teaching_pairs == 1
+    refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"]
+    assert [item["pairs"] for item in refusals] == [["holdout-one"]]
 
 
 def test_the_mechanism_lever_refuses_a_candidate_a_holdout_pair_taught(tmp_path: Path) -> None:
-    from openultrasast.improve.evolve import propose_mechanism_edits
-    from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
-    from openultrasast.semantic.variants import Shape
+    """Both arms. The shape is the one the corpus really teaches, taken from the pair itself.
 
-    shape = Shape(
-        language="python",
-        sink_name="system",
-        arity=1,
-        source_positions=(0,),
-        source_kinds=("request",),
-        guard="none",
-        mechanism="source_reaches_sink",
-    )
-    candidates = MechanismStore(tmp_path / "candidates.jsonl")
-    append_from_pair(candidates, shape, summary="s", cwe="CWE-78", pair="holdout-one", provenance="human", tier="seeded")
-    cases = [_case(tmp_path, "train-one"), _case(tmp_path, "holdout-one", split="holdout")]
-    edits = propose_mechanism_edits(cases, candidates, tmp_path / "scan.jsonl")
-    assert [edit.mechanism_id for edit in edits if edit.action == "admit"] == []
+    Asserting only the refusal would pass against a lever that admits nothing at all, and a hand-written shape that
+    matches no excerpt can never be admitted either, so neither would measure the split rule (Req 5.2)."""
+    from openultrasast.improve.evolve import propose_mechanism_edits
+    from openultrasast.semantic.loo import _facts
+    from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
+    from openultrasast.semantic.seed import pair_lessons
+
+    cases = [_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")]
+    (lesson,) = pair_lessons(cases[0], _facts())
+
+    def store(name: str, pair: str) -> MechanismStore:
+        path = MechanismStore(tmp_path / name)
+        append_from_pair(path, lesson.shape, summary=lesson.summary, cwe=lesson.cwe, pair=pair, provenance="human", tier="seeded")
+        return path
+
+    reasons: list[dict[str, object]] = []
+    refused = propose_mechanism_edits(cases, store("refused.jsonl", "holdout-one"), tmp_path / "scan.jsonl", degradations=reasons)
+    assert [edit.mechanism_id for edit in refused if edit.action == "admit"] == []
+    assert [item["pairs"] for item in reasons] == [["holdout-one"]]
+    admitted = propose_mechanism_edits(cases, store("allowed.jsonl", "train-one"), tmp_path / "scan2.jsonl")
+    assert [edit.mechanism_id for edit in admitted if edit.action == "admit"], "a train-taught shape is still admissible"
 
 
 def test_the_corrected_lever_number_is_measured_and_committed() -> None:

tests/test_obligation_export.py

@@ -128,8 +128,12 @@ def test_the_train_split_of_vibe_py_teaches_no_obligation_shape_today(tmp_path:
     cases = select_vendored(select_slice(load_pair_catalog(), "vibe-py"))
     report = export_mechanisms(cases, store)
     assert obligation_mechanisms(store.load()) == ()
-    refused = {pair for pair, reason in report.skipped if reason.startswith("split:")}
-    assert {"vampi-books-get-by-title", "vampi-users-update-password", "threatbyte-api-v1-get"} <= refused
+    refused = {pair: reason for pair, reason in report.skipped}
+    assert refused["vampi-users-update-password"] == "split:holdout"
+    assert refused["threatbyte-api-v1-get"] == "split:holdout"
+    # Its own reason, not its split: this row is a byte-identical twin that happens to sit on the holdout side,
+    # and reporting `split:holdout` would hide a corpus fact behind a split fact (Req 5.2).
+    assert refused["vampi-books-get-by-title"] == "unscorable:identical_twin"
     flow_rows = [row for row in store.load() if row.shape is not None and row.shape.get("family", "sink") == "sink"]
     assert report.records == len(flow_rows) and len(flow_rows) >= 1  # the flow family still teaches from the train split