fix(learning-harness): the round-4 review findings

Three defects, two of them the same shape as ones already rejected once:

- The meta-agent proposer still could not propose. `MetaAgent(inner_model=)`
  takes a ModelConfig; it was handed a model name, a `# type: ignore` hid
  that from mypy, and a bare `except Exception` turned the resulting
  AttributeError into `meta_agent_failed` -- which reads exactly like the
  model declining. The composition is now a seam a test can pin, and a
  construction error is re-raised rather than reported as a quiet round.
- `ousast pairs --hunter` crashed in its documented form: the K floor landed
  with `--k-runs` still defaulting to 1. The covering test always passed
  `--k-runs 4`, which is why nothing noticed.
- 28 rows (12% of the corpus) resolved a span on the vulnerable side and
  none on the fixed side, so every fixed-side finding read as outside the
  labeled function and the pair scored as silence whatever the detector
  said. `fix_function` now reaches the catalog and the scorer, and the
  generator derives it from the fixed excerpt for the 25 trap rows that
  never declared one. Measured after: 0 rows.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/learning/proposer.py

@@ -201,6 +201,10 @@ class HarnessXProposer:
             before = _directory_text(workspace)
             try:
                 hypothesis = call(workspace=workspace, facts=facts, model=self.model)
+            except (TypeError, AttributeError, ImportError):
+                # "we called the agent wrong" is not "the agent declined", and folding one into the other is how a
+                # broken composition reads as a quiet round for as long as nobody looks.
+                raise
             except Exception as exc:  # noqa: BLE001 — an agent that failed proposed nothing, and the round says why
                 self.reason = f"meta_agent_failed: {type(exc).__name__}"
                 return None
@@ -239,6 +243,26 @@ def _directory_text(directory: Path) -> dict[str, str]:
     return out
 
 
+def _meta_agent_class() -> type:
+    """`MetaAgent` behind `harness_ext`. A seam, so the argument contract can be pinned without the extra."""
+    from ..harness_ext import require_harnessx
+
+    require_harnessx()
+    from harnessx.meta_harness.agent import MetaAgent  # type: ignore[import-not-found]
+
+    return MetaAgent  # type: ignore[no-any-return]
+
+
+def _model_config(model: str, provider: str = "anthropic") -> object:
+    """`MetaAgent(inner_model=...)` takes a `ModelConfig`, never a model name; the name goes to the provider."""
+    from ..harness_ext import build_provider, require_harnessx
+
+    require_harnessx()
+    from harnessx.core.model_config import ModelConfig  # type: ignore[import-not-found]
+
+    return ModelConfig(main=build_provider(model, provider))
+
+
 def _meta_agent_call() -> MetaAgentCall:
     """Compose `MetaAgent.evolve` behind `harness_ext`, writing only inside the scratch copy.
 
@@ -252,12 +276,10 @@ def _meta_agent_call() -> MetaAgentCall:
     def call(*, workspace: Path, facts: FailureFacts, model: str) -> str:
         import asyncio
 
-        from harnessx.meta_harness.agent import MetaAgent  # type: ignore[import-not-found]
-
         trajectories = workspace.parent / "trajectories"
         trajectories.mkdir(parents=True, exist_ok=True)
         (trajectories / "facts.json").write_text(facts_prompt(facts), encoding="utf-8")
-        agent = MetaAgent(inner_model=model, allowed_write_roots=(workspace,))  # type: ignore[arg-type]
+        agent = _meta_agent_class()(inner_model=_model_config(model), allowed_write_roots=(workspace,))
         output = workspace.parent / "meta-out"
         output.mkdir(parents=True, exist_ok=True)
         asyncio.run(agent.evolve(current_config=workspace, trajectories_dir=trajectories, output_dir=output))

src/openultrasast/learning/rounds.py

@@ -457,11 +457,13 @@ def _pairs_named_by(proposal: Proposal, cases: Sequence[PairCase]) -> tuple[str,
     A memory entry that quotes a held-out pair is the loop learning the answer instead of the shape, and it looks
     exactly like a legitimate hard negative until the holdout number is read as if it meant something."""
     haystack = "\n".join([*proposal.predicted_affected, *proposal.predicted_at_risk, *proposal.change.values()])
-    named = []
-    for case in cases:
-        name = str(getattr(case, "name", ""))
-        if name and re.search(rf"(?<![\w-]){re.escape(name)}(?![\w-])", haystack):
-            named.append(name)
+    # Substring first, regex only for the handful that survive it: the corpus is a few hundred rows and a round
+    # runs this once, but a per-round sweep of one regex per row is the kind of thing that stops being free.
+    named = [
+        name
+        for case in cases
+        if (name := str(getattr(case, "name", ""))) and name in haystack and re.search(rf"(?<![\w-]){re.escape(name)}(?![\w-])", haystack)
+    ]
     return tuple(sorted(set(named)))
 
 

src/openultrasast/learning/scoring.py

@@ -34,9 +34,9 @@ def is_post_cutoff(fix_date: str, cutoff: str) -> bool:
     A year-precision row counts only when the whole year is after the cutoff, so a coarse date can under-claim the
     clean slice but never over-claim it — and over-claiming is the one that would turn contamination into a result.
     """
-    if not fix_date or not cutoff:
-        return False
-    earliest = fix_date if len(fix_date) == 10 else f"{fix_date[:4]}-01-01"
+    if not fix_date or len(cutoff) != 10:
+        return False  # a cutoff coarser than a day is the one over-claim path there is, so it is refused outright
+    earliest = fix_date if len(fix_date) == 10 else f"{fix_date}-01-01"[:10]
     return earliest > cutoff
 
 
@@ -96,7 +96,22 @@ def score_pair_family(
             unscorable_reason=reason,
         )
     spans = _spans(case, ranges)
-    fixed_spans = _spans(case, fixed_ranges) if fixed_ranges is not None else spans
+    if fixed_ranges is None:
+        fixed_spans = spans
+    else:
+        # A trap pair's fixed side holds a different, correctly guarded handler; `fix_function` names it. Without a
+        # span there, every fixed-side finding reads as outside the labeled function and the pair scores as silence.
+        fixed_spans = _spans(case, fixed_ranges, function=str(getattr(case, "fix_function", "") or ""))
+        if not fixed_spans:
+            return PairFamilyScore(
+                pair=name,
+                family=family,
+                slice=slice_name,
+                fix_date=str(getattr(case, "fix_date", "") or ""),
+                runs=(),
+                outcome="unscorable",
+                unscorable_reason="unresolved_label:fixed",
+            )
     outcomes: list[FamilyOutcome] = []
     other_vuln = other_fixed = fabricated_vuln = fabricated_fixed = 0
     rung: Rung = "suspicion"
@@ -112,7 +127,15 @@ def score_pair_family(
         rung = _best_rung(rung, hits_vuln)
         outcomes.append(_outcome(bool(hits_vuln), bool(hits_fixed)))
     if not outcomes:
-        return PairFamilyScore(pair=name, family=family, slice=slice_name, runs=(), outcome="unscorable", unscorable_reason="no_runs")
+        return PairFamilyScore(
+            pair=name,
+            family=family,
+            slice=slice_name,
+            fix_date=str(getattr(case, "fix_date", "") or ""),
+            runs=(),
+            outcome="unscorable",
+            unscorable_reason="no_runs",
+        )
     majority = Counter(outcomes).most_common(1)[0][0]
     return PairFamilyScore(
         pair=name,
@@ -170,9 +193,9 @@ def unscorable_reason(
     return None
 
 
-def _spans(case: object, ranges: FunctionRanges) -> tuple[tuple[str, int, int], ...]:
-    """The (path, start, end) spans of the labeled functions, from the parsed vulnerable side."""
-    wanted = {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()}
+def _spans(case: object, ranges: FunctionRanges, *, function: str = "") -> tuple[tuple[str, int, int], ...]:
+    """The (path, start, end) spans of the labeled functions. ``function`` overrides the label for the fixed side."""
+    wanted = {function} if function else {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()}
     wanted.discard("")
     found: list[tuple[str, int, int]] = []
     for path, entries in ranges.items():