feat(learning-harness): fix dates and the post-cutoff slice

The catalog generator fills `fix_date` from the recipe at the precision the
recipe has -- a full ISO day when one was recorded, otherwise the year.
Every harvested recipe carries a year and almost none carries a day, and
writing `-01-01` would turn a year into a claim about a month. A row counts
as post-cutoff only when its earliest possible date is after the cutoff, so
a coarse date can under-claim the clean slice but never over-claim it.

Each family now reports the same numbers over the rows fixed after the
cutoff, beside the whole number and never instead of it, with the undated
count on its own line. Without a configured cutoff every scoreboard says so
rather than inventing one: guessing a model's training cutoff and then
publishing a contamination number from the guess would be worse than
reporting none.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/cli.py

@@ -1288,12 +1288,22 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
             for case in cases
             if _label(case, taxonomy) in configs
         ]
-        families = {name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy).items()}
+        cutoff = config.learning.cutoff_date or ""
+        families = {name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, cutoff=cutoff).items()}
         if args.json:
             print(json.dumps({"families": families}, indent=2, sort_keys=True))
         else:
             for name, block in sorted(families.items()):
-                print(f"learning score {name}: scorable={block['scorable']} recall={block['recall']:.3f} youden={block['youden']:.3f}")
+                raw = block.get("post_cutoff")
+                recent: dict[str, object] = raw if isinstance(raw, dict) else {}
+                tail = (
+                    f" post-cutoff({cutoff})={recent.get('scorable', 0)} undated={block['undated']}"
+                    if cutoff
+                    else " post-cutoff=none configured"
+                )
+                print(
+                    f"learning score {name}: scorable={block['scorable']} recall={block['recall']:.3f} youden={block['youden']:.3f}{tail}"
+                )
         return 0
     if args.learning_command == "baseline":
         report = run_baseline(
@@ -1304,6 +1314,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
             model=model,
             out_dir=out,
             k_runs=k_runs,
+            cutoff=config.learning.cutoff_date or "",
         )
         print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}")
         return 0

src/openultrasast/learning/publish.py

@@ -131,11 +131,47 @@ def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Map
                 f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |"
             )
     lines += ["", _credit_note(models), ""]
+    cutoff = _cutoff_note(models, families)
+    if cutoff:
+        lines += [cutoff, ""]
     if lever is not None:
         lines += [_lever_note(lever), ""]
     return "\n".join(lines)
 
 
+def _cutoff_note(models: Mapping[str, Baseline], families: tuple[str, ...]) -> str:
+    """The post-cutoff slice, its cutoff and the undated count (Req 10.5).
+
+    A model that may have read the fix cannot be measured on it the same way as one that cannot have, so the rows
+    fixed after the detector's training cutoff are reported apart. Rows the corpus records no date for are in
+    neither slice, and their number is here rather than folded silently into the whole."""
+    rows = [
+        (model, family, baseline.metrics[family])
+        for model, baseline in sorted(models.items())
+        for family in families
+        if family in baseline.metrics and str(baseline.metrics[family].get("cutoff") or "")
+    ]
+    if not rows:
+        return "No detector cutoff is configured, so no post-cutoff slice is reported; every figure above is over the whole corpus."
+    cutoff = str(rows[0][2].get("cutoff"))
+    lines = [
+        f"Post-cutoff slice, fixed after {cutoff} and therefore beyond the detector's training data. "
+        "A row counts only when its earliest possible date is after the cutoff, so a year-precision date can "
+        "under-claim this slice but never over-claim it.",
+        "",
+        "| model | family | post-cutoff scorable | post-cutoff recall | post-cutoff Youden | undated |",
+        "|---|---|---|---|---|---|",
+    ]
+    for model, family, row in rows:
+        block = row.get("post_cutoff")
+        block = block if isinstance(block, Mapping) else {}
+        lines.append(
+            f"| {model} | {family} | {block.get('scorable', 0)} | {_num(block.get('recall'))} "
+            f"| {_num(block.get('youden'))} | {row.get('undated', 0)} |"
+        )
+    return "\n".join(lines)
+
+
 def _credit_note(models: Mapping[str, Baseline]) -> str:
     hierarchical = any(bool(row.get("hierarchical_credit")) for baseline in models.values() for row in baseline.metrics.values())
     if hierarchical:

src/openultrasast/learning/rounds.py

@@ -99,6 +99,7 @@ def run_baseline(
     k_runs: int = DEFAULT_K_RUNS,
     prompt: str | None = None,
     spent_usd: Callable[[], float] | None = None,
+    cutoff: str = "",
 ) -> BaselineReport:
     """Clone one detector into every family, measure each family **with its own configuration**, write the artifacts.
 
@@ -123,7 +124,7 @@ def run_baseline(
         if config is None:
             continue  # a family with no configuration is not measured rather than measured with someone else's
         scores.append(score_case(case, scan_factory(config), taxonomy=taxonomy, runs=k_runs, family=name))
-    metrics = aggregate(scores, taxonomy=taxonomy)
+    metrics = aggregate(scores, taxonomy=taxonomy, cutoff=cutoff)
     floors = {name: _floor(name, block, [item for item in scores if item.family == name], k_runs) for name, block in metrics.items()}
     report = BaselineReport(
         model=model,
@@ -132,7 +133,7 @@ def run_baseline(
         slices=tuple(slices),
         floors=floors,
         metrics={name: block.to_dict() for name, block in metrics.items()},
-        per_slice={name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, per_slice=True).items()},
+        per_slice={name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, per_slice=True, cutoff=cutoff).items()},
         scores=tuple(scores),
         cost_usd=spent_usd() if spent_usd is not None else 0.0,
     )
@@ -156,6 +157,7 @@ def score_case(case: PairCase, scan: Scan, *, taxonomy: FamilyTaxonomy, runs: in
             runs=(),
             outcome="unscorable",
             slice=case.slice,
+            fix_date=case.fix_date,
             unscorable_reason=case.unscorable,
         )
     with tempfile.TemporaryDirectory(prefix="ousast-round-") as scratch:

src/openultrasast/learning/scoring.py

@@ -25,6 +25,21 @@ FamilyOutcome = Literal["pair_correct", "both_flagged", "both_silent", "reversed
 # Families whose bug shape is the absence of a check on a reached handler: without the registration that
 # makes the handler reachable, the pair says nothing either way, which is a corpus defect with its own name.
 CONTEXT_DEPENDENT_FAMILIES = frozenset({"access_control"})
+
+
+def is_post_cutoff(fix_date: str, cutoff: str) -> bool:
+    """True when the *earliest possible* date of ``fix_date`` is after ``cutoff`` (Req 10.5).
+
+    The corpus records dates at the precision it has: a full ISO day for a few rows, a year for almost all of them.
+    A year-precision row counts only when the whole year is after the cutoff, so a coarse date can under-claim the
+    clean slice but never over-claim it — and over-claiming is the one that would turn contamination into a result.
+    """
+    if not fix_date or not cutoff:
+        return False
+    earliest = fix_date if len(fix_date) == 10 else f"{fix_date[:4]}-01-01"
+    return earliest > cutoff
+
+
 Rung = Literal["suspicion", "static_corroboration", "proven"]
 _RUNGS: tuple[Rung, ...] = ("suspicion", "static_corroboration", "proven")
 FunctionRanges = Mapping[str, Sequence[tuple[str, int, int]]]
@@ -46,6 +61,7 @@ class PairFamilyScore:
     flips: int = 0  # runs that disagreed with the majority
     rung: Rung = "suspicion"
     unscorable_reason: str | None = None
+    fix_date: str = ""  # the date the corpus recorded, at whatever precision it has (Req 10.5)
 
 
 def score_pair_family(
@@ -70,7 +86,15 @@ def score_pair_family(
     slice_name = str(getattr(case, "slice", "") or "")
     reason = unscorable_reason(case, parse_ok=parse_ok, ranges=ranges, entry_points=entry_points, family=family)
     if reason is not None:
-        return PairFamilyScore(pair=name, family=family, slice=slice_name, runs=(), outcome="unscorable", unscorable_reason=reason)
+        return PairFamilyScore(
+            pair=name,
+            family=family,
+            slice=slice_name,
+            fix_date=str(getattr(case, "fix_date", "") or ""),
+            runs=(),
+            outcome="unscorable",
+            unscorable_reason=reason,
+        )
     spans = _spans(case, ranges)
     fixed_spans = _spans(case, fixed_ranges) if fixed_ranges is not None else spans
     outcomes: list[FamilyOutcome] = []
@@ -94,6 +118,7 @@ def score_pair_family(
         pair=name,
         family=family,
         slice=slice_name,
+        fix_date=str(getattr(case, "fix_date", "") or ""),
         runs=tuple(outcomes),
         outcome=majority,
         other_findings_vuln=other_vuln,
@@ -219,6 +244,9 @@ class FamilyMetrics:
     p_value: float | None = None  # two-sided sign test over the discordant pairs
     reliable_change: bool = False
     hierarchical_credit: bool = False  # False: the taxonomy is flat, so no answer could earn partial credit (Req 3.7)
+    cutoff: str = ""  # the detector cutoff these numbers were split on, or "" when none is configured (Req 10.5)
+    undated: int = 0  # rows the corpus records no date for; they are in neither slice and are counted here
+    post_cutoff: dict[str, object] = field(default_factory=dict)  # the same numbers over rows fixed after the cutoff
 
     def to_dict(self) -> dict[str, object]:
         return {
@@ -238,6 +266,9 @@ class FamilyMetrics:
             "p_value": self.p_value,
             "reliable_change": self.reliable_change,
             "hierarchical_credit": self.hierarchical_credit,
+            "cutoff": self.cutoff,
+            "undated": self.undated,
+            "post_cutoff": dict(self.post_cutoff),
         }
 
 
@@ -249,8 +280,13 @@ def aggregate(
     fpr_ceiling: float = 0.1,
     per_slice: bool = False,
     alpha: float = 0.05,
+    cutoff: str = "",
 ) -> dict[str, FamilyMetrics]:
-    """Group scores into per-family numbers. Unscorable rows are listed by reason and leave every denominator."""
+    """Group scores into per-family numbers. Unscorable rows are listed by reason and leave every denominator.
+
+    With a ``cutoff`` each family also reports the same numbers over the rows fixed after it, beside the whole
+    number and never instead of it: a model that may have read the fix cannot be measured on it the same way as one
+    that cannot have. Rows the corpus records no date for are counted on their own (Req 10.5)."""
     grouped: dict[str, list[PairFamilyScore]] = {}
     for score in scores:
         grouped.setdefault(f"{score.slice}/{score.family}" if per_slice else score.family, []).append(score)
@@ -281,6 +317,7 @@ def aggregate(
             elif before != "pair_correct" and item.outcome == "pair_correct":
                 positive += 1
         p_value = sign_test(positive, negative) if baseline is not None else None
+        recent = [item for item in scorable if is_post_cutoff(item.fix_date, cutoff)] if cutoff else []
         out[key] = FamilyMetrics(
             family=group[0].family,
             taxonomy_version=taxonomy.version,
@@ -298,10 +335,28 @@ def aggregate(
             p_value=p_value,
             reliable_change=bool(p_value is not None and p_value < alpha and positive != negative),
             hierarchical_credit=taxonomy.hierarchical,
+            cutoff=cutoff,
+            undated=sum(1 for item in group if not item.fix_date) if cutoff else 0,
+            post_cutoff=_slice_numbers(recent) if cutoff else {},
         )
     return out
 
 
+def _slice_numbers(scores: Sequence[PairFamilyScore]) -> dict[str, object]:
+    """Recall, silence and Youden over a subset, with the subset's own denominator beside them."""
+    total = len(scores)
+    counts = Counter(item.outcome for item in scores)
+    recall = (counts["pair_correct"] + counts["both_flagged"]) / total if total else 0.0
+    silence = (counts["pair_correct"] + counts["both_silent"]) / total if total else 0.0
+    return {
+        "scorable": total,
+        "outcomes": {name: count for name, count in sorted(counts.items())},
+        "recall": recall,
+        "silence": silence,
+        "youden": (recall + silence - 1.0) if total else 0.0,
+    }
+
+
 def sign_test(better: int, worse: int) -> float:
     """Two-sided exact sign test over the pairs that changed. 1.0 when nothing changed, so noise never reads as progress."""
     total = better + worse
@@ -320,6 +375,7 @@ __all__ = [
     "PairFamilyScore",
     "Rung",
     "aggregate",
+    "is_post_cutoff",
     "score_pair_family",
     "sign_test",
     "unscorable_reason",