fix(learning-harness): the round-4 review findings

Three defects, two of them the same shape as ones already rejected once:

- The meta-agent proposer still could not propose. `MetaAgent(inner_model=)`
  takes a ModelConfig; it was handed a model name, a `# type: ignore` hid
  that from mypy, and a bare `except Exception` turned the resulting
  AttributeError into `meta_agent_failed` -- which reads exactly like the
  model declining. The composition is now a seam a test can pin, and a
  construction error is re-raised rather than reported as a quiet round.
- `ousast pairs --hunter` crashed in its documented form: the K floor landed
  with `--k-runs` still defaulting to 1. The covering test always passed
  `--k-runs 4`, which is why nothing noticed.
- 28 rows (12% of the corpus) resolved a span on the vulnerable side and
  none on the fixed side, so every fixed-side finding read as outside the
  labeled function and the pair scored as silence whatever the detector
  said. `fix_function` now reaches the catalog and the scorer, and the
  generator derives it from the fixed excerpt for the 25 trap rows that
  never declared one. Measured after: 0 rows.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/cli.py

@@ -189,7 +189,9 @@ def main(argv: list[str] | None = None) -> int:
     )
     pairs.add_argument("--loo-out", type=Path, default=None, help="where to write loo.json (default: reports/loo.json)")
     pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON")
-    pairs.add_argument("--k-runs", type=int, default=1, help="runs per pair on the hunter path; the family scorer needs at least three")
+    pairs.add_argument(
+        "--k-runs", type=int, default=None, help="runs per pair on the hunter path; never fewer than three, and three by default"
+    )
 
     subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration")
     mechanisms = subparsers.add_parser("mechanisms", help="maintainer: mechanism memory seeded from trusted pairs")
@@ -982,7 +984,10 @@ def _pairs(
             from .learning.families import load_families
 
             scan = make_hunter_scan(client, model, taxonomy=load_families())
-    result = evaluate_catalog(cases, hunter=scan, pointers=True if pointers else None, k_runs=k_runs)
+    try:
+        result = evaluate_catalog(cases, hunter=scan, pointers=True if pointers else None, k_runs=k_runs)
+    except ValueError as exc:
+        raise SystemExit(f"--k-runs: {exc}") from exc
     if json_out:
         print(json.dumps(result_payload(result), indent=2, sort_keys=True))
         return 0

src/openultrasast/learning/proposer.py

@@ -201,6 +201,10 @@ class HarnessXProposer:
             before = _directory_text(workspace)
             try:
                 hypothesis = call(workspace=workspace, facts=facts, model=self.model)
+            except (TypeError, AttributeError, ImportError):
+                # "we called the agent wrong" is not "the agent declined", and folding one into the other is how a
+                # broken composition reads as a quiet round for as long as nobody looks.
+                raise
             except Exception as exc:  # noqa: BLE001 — an agent that failed proposed nothing, and the round says why
                 self.reason = f"meta_agent_failed: {type(exc).__name__}"
                 return None
@@ -239,6 +243,26 @@ def _directory_text(directory: Path) -> dict[str, str]:
     return out
 
 
+def _meta_agent_class() -> type:
+    """`MetaAgent` behind `harness_ext`. A seam, so the argument contract can be pinned without the extra."""
+    from ..harness_ext import require_harnessx
+
+    require_harnessx()
+    from harnessx.meta_harness.agent import MetaAgent  # type: ignore[import-not-found]
+
+    return MetaAgent  # type: ignore[no-any-return]
+
+
+def _model_config(model: str, provider: str = "anthropic") -> object:
+    """`MetaAgent(inner_model=...)` takes a `ModelConfig`, never a model name; the name goes to the provider."""
+    from ..harness_ext import build_provider, require_harnessx
+
+    require_harnessx()
+    from harnessx.core.model_config import ModelConfig  # type: ignore[import-not-found]
+
+    return ModelConfig(main=build_provider(model, provider))
+
+
 def _meta_agent_call() -> MetaAgentCall:
     """Compose `MetaAgent.evolve` behind `harness_ext`, writing only inside the scratch copy.
 
@@ -252,12 +276,10 @@ def _meta_agent_call() -> MetaAgentCall:
     def call(*, workspace: Path, facts: FailureFacts, model: str) -> str:
         import asyncio
 
-        from harnessx.meta_harness.agent import MetaAgent  # type: ignore[import-not-found]
-
         trajectories = workspace.parent / "trajectories"
         trajectories.mkdir(parents=True, exist_ok=True)
         (trajectories / "facts.json").write_text(facts_prompt(facts), encoding="utf-8")
-        agent = MetaAgent(inner_model=model, allowed_write_roots=(workspace,))  # type: ignore[arg-type]
+        agent = _meta_agent_class()(inner_model=_model_config(model), allowed_write_roots=(workspace,))
         output = workspace.parent / "meta-out"
         output.mkdir(parents=True, exist_ok=True)
         asyncio.run(agent.evolve(current_config=workspace, trajectories_dir=trajectories, output_dir=output))

src/openultrasast/learning/rounds.py

@@ -457,11 +457,13 @@ def _pairs_named_by(proposal: Proposal, cases: Sequence[PairCase]) -> tuple[str,
     A memory entry that quotes a held-out pair is the loop learning the answer instead of the shape, and it looks
     exactly like a legitimate hard negative until the holdout number is read as if it meant something."""
     haystack = "\n".join([*proposal.predicted_affected, *proposal.predicted_at_risk, *proposal.change.values()])
-    named = []
-    for case in cases:
-        name = str(getattr(case, "name", ""))
-        if name and re.search(rf"(?<![\w-]){re.escape(name)}(?![\w-])", haystack):
-            named.append(name)
+    # Substring first, regex only for the handful that survive it: the corpus is a few hundred rows and a round
+    # runs this once, but a per-round sweep of one regex per row is the kind of thing that stops being free.
+    named = [
+        name
+        for case in cases
+        if (name := str(getattr(case, "name", ""))) and name in haystack and re.search(rf"(?<![\w-]){re.escape(name)}(?![\w-])", haystack)
+    ]
     return tuple(sorted(set(named)))
 
 

src/openultrasast/learning/scoring.py

@@ -34,9 +34,9 @@ def is_post_cutoff(fix_date: str, cutoff: str) -> bool:
     A year-precision row counts only when the whole year is after the cutoff, so a coarse date can under-claim the
     clean slice but never over-claim it — and over-claiming is the one that would turn contamination into a result.
     """
-    if not fix_date or not cutoff:
-        return False
-    earliest = fix_date if len(fix_date) == 10 else f"{fix_date[:4]}-01-01"
+    if not fix_date or len(cutoff) != 10:
+        return False  # a cutoff coarser than a day is the one over-claim path there is, so it is refused outright
+    earliest = fix_date if len(fix_date) == 10 else f"{fix_date}-01-01"[:10]
     return earliest > cutoff
 
 
@@ -96,7 +96,22 @@ def score_pair_family(
             unscorable_reason=reason,
         )
     spans = _spans(case, ranges)
-    fixed_spans = _spans(case, fixed_ranges) if fixed_ranges is not None else spans
+    if fixed_ranges is None:
+        fixed_spans = spans
+    else:
+        # A trap pair's fixed side holds a different, correctly guarded handler; `fix_function` names it. Without a
+        # span there, every fixed-side finding reads as outside the labeled function and the pair scores as silence.
+        fixed_spans = _spans(case, fixed_ranges, function=str(getattr(case, "fix_function", "") or ""))
+        if not fixed_spans:
+            return PairFamilyScore(
+                pair=name,
+                family=family,
+                slice=slice_name,
+                fix_date=str(getattr(case, "fix_date", "") or ""),
+                runs=(),
+                outcome="unscorable",
+                unscorable_reason="unresolved_label:fixed",
+            )
     outcomes: list[FamilyOutcome] = []
     other_vuln = other_fixed = fabricated_vuln = fabricated_fixed = 0
     rung: Rung = "suspicion"
@@ -112,7 +127,15 @@ def score_pair_family(
         rung = _best_rung(rung, hits_vuln)
         outcomes.append(_outcome(bool(hits_vuln), bool(hits_fixed)))
     if not outcomes:
-        return PairFamilyScore(pair=name, family=family, slice=slice_name, runs=(), outcome="unscorable", unscorable_reason="no_runs")
+        return PairFamilyScore(
+            pair=name,
+            family=family,
+            slice=slice_name,
+            fix_date=str(getattr(case, "fix_date", "") or ""),
+            runs=(),
+            outcome="unscorable",
+            unscorable_reason="no_runs",
+        )
     majority = Counter(outcomes).most_common(1)[0][0]
     return PairFamilyScore(
         pair=name,
@@ -170,9 +193,9 @@ def unscorable_reason(
     return None
 
 
-def _spans(case: object, ranges: FunctionRanges) -> tuple[tuple[str, int, int], ...]:
-    """The (path, start, end) spans of the labeled functions, from the parsed vulnerable side."""
-    wanted = {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()}
+def _spans(case: object, ranges: FunctionRanges, *, function: str = "") -> tuple[tuple[str, int, int], ...]:
+    """The (path, start, end) spans of the labeled functions. ``function`` overrides the label for the fixed side."""
+    wanted = {function} if function else {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()}
     wanted.discard("")
     found: list[tuple[str, int, int]] = []
     for path, entries in ranges.items():

src/openultrasast/pairs.py

@@ -131,6 +131,12 @@ class PairCase:
     unscorable: str | None = None  # computed: identical_twin, known_limit:<reason> (learning-harness Req 4.1)
     split_declared: bool = True  # False when the catalog row named no split, so the split helper may assign one
     fix_date: str = ""  # ISO date of the fixing commit; the catalog generator fills it, the split helper orders by it
+    # The function the *fixed* side actually holds, when it is not the labeled one. Real-Vuln-Benchmark traps take
+    # their fixed side from a different, correctly guarded handler of the same snapshot, and without this the fixed
+    # side resolves no span at all: every finding there reads as outside the labeled function, so the pair scores as
+    # silence rather than as a leak (learning-harness Req 3.2, 4.1).
+    fix_function: str = ""
+
     recipe: tuple[tuple[str, object], ...] = ()  # pointer pairs carry their harvest recipe instead of excerpts
     # (relpath, vuln document, fixed document): files that register the handler from outside its own module — an
     # OpenAPI operation, a router table. Materialized beside the excerpt so the mapper can name the handler at all
@@ -292,6 +298,7 @@ def _load_catalog_file(path: Path) -> tuple[PairCase, ...]:
                 unscorable=_unscorable_reason(known_limit, vuln_file, fixed_file, vendored=vendored),
                 split_declared="split" in item,
                 fix_date=str(item.get("fix_date", "")),
+                fix_function=str(item.get("fix_function", "")),
                 review_tier=review_tier,
                 reviewer=reviewer,
                 vendored=vendored,
@@ -625,8 +632,10 @@ def _body_digest(path: Path) -> str | None:
     return hashlib.sha256(body.encode()).hexdigest() if body else None
 
 
-# The reasons a row may carry verbatim from the catalog, because the loader computes them under the same names.
-UNSCORABLE_REASONS = frozenset({"identical_twin"})
+# The reasons a row may carry verbatim from the catalog, because something else computes them under the same
+# names. A maintainer declaring one of these is stating the fact the computation would have found, so the two must
+# not end up as two buckets in the same report (learning-harness Req 4.1, 10.3).
+UNSCORABLE_REASONS = frozenset({"identical_twin", "unsupported_language", "unresolved_label", "missing_context"})
 
 
 def _unscorable_reason(known_limit: object, vuln_file: Path, fixed_file: Path, *, vendored: bool) -> str | None:

src/openultrasast/semantic/loo.py

@@ -93,20 +93,31 @@ def evaluate_loo(cases: Sequence[PairCase], *, facts: SemanticFacts | None = Non
     targets: list[PairCase] = []
     skipped: list[tuple[str, str]] = []
     for case in cases:
-        if case.known_limit:
-            skipped.append((case.name, f"known_limit:{case.known_limit}"))
+        if case.unscorable or case.known_limit:
+            # `PairCase.unscorable` verbatim when the loader computed one: one spelling per corpus fact, and it
+            # already carries the `known_limit:` prefix when the limit is a maintainer's own word rather than one
+            # of the vocabulary's. A case built by hand may carry only the declaration.
+            skipped.append((case.name, case.unscorable or f"known_limit:{case.known_limit}"))
         elif not case.vuln_file.is_file() or not case.fixed_file.is_file():
             skipped.append((case.name, "pointer_pair_not_cached" if not case.vendored else "excerpt_missing"))
         else:
             targets.append(case)
-    from ..learning.split import is_teacher, refuse_if_holdout
+    from ..learning.split import GATING_TIERS, holdout_names, is_teacher, refuse_if_holdout
 
     # learning-harness Req 5.1: every pair is still scored, but only a train-split teacher may seed a store.
     lessons = {case.name: pair_lessons(case, loaded) for case in targets if is_teacher(case)}
     outcomes = [_hold_out(case, targets, lessons, loaded) for case in targets]
-    # Req 5.2: the pairs that were kept out of the teaching set are named, not silently dropped.
-    refusal = refuse_if_holdout([case.name for case in targets if case.name not in lessons], targets)
+    # Req 5.2: the holdout pairs kept out of the teaching set are named, not silently dropped. Non-teachers held
+    # back for their tier or their unscorable reason are already in `skipped` with that reason.
+    excluded = [case.name for case in targets if case.name not in lessons]
+    refusal = refuse_if_holdout(excluded, targets)
     refusals = (refusal.degradation(),) if refusal is not None else ()
+    withheld = tuple(
+        (case.name, "tier" if case.review_tier not in GATING_TIERS else "unscorable")
+        for case in targets
+        if case.name in set(excluded) - set(holdout_names(targets))
+    )
+    skipped.extend((name, f"not_a_teacher:{why}") for name, why in withheld)
     return LooResult(
         outcomes=tuple(outcomes),
         per_slice=_group(outcomes, lambda item: (item.slice,)),

src/openultrasast/semantic/seed.py

@@ -197,12 +197,12 @@ def _skip_reason(case: PairCase) -> str | None:
 
     if case.review_tier not in GATING_TIERS:
         return f"tier:{case.review_tier}"
-    if case.unscorable:
-        # One spelling per corpus fact. `identical_twin` declared in the catalog and `identical_twin` computed from
-        # the bytes are the same reason, and calling one of them `split:train` is simply untrue (Req 5.1, 10.3).
-        return f"unscorable:{case.unscorable}"
-    if case.known_limit:
-        return f"known_limit:{case.known_limit}"
+    if case.unscorable or case.known_limit:
+        # `PairCase.unscorable` verbatim when the loader computed one, the same string every other path reports:
+        # `identical_twin` declared in the catalog and `identical_twin` computed from the bytes are one fact, and
+        # calling either of them `split:train` is simply untrue. A case built by hand may carry only the
+        # declaration (Req 5.1, 10.3).
+        return case.unscorable or f"known_limit:{case.known_limit}"
     if not is_teacher(case) and case.vendored:
         return f"split:{case.split}"
     if not case.vuln_file.is_file() or not case.fixed_file.is_file():