mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 20:49:28 +00:00
feat(learning-harness): fix dates and the post-cutoff slice
The catalog generator fills `fix_date` from the recipe at the precision the recipe has -- a full ISO day when one was recorded, otherwise the year. Every harvested recipe carries a year and almost none carries a day, and writing `-01-01` would turn a year into a claim about a month. A row counts as post-cutoff only when its earliest possible date is after the cutoff, so a coarse date can under-claim the clean slice but never over-claim it. Each family now reports the same numbers over the rows fixed after the cutoff, beside the whole number and never instead of it, with the undated count on its own line. Without a configured cutoff every scoreboard says so rather than inventing one: guessing a model's training cutoff and then publishing a contamination number from the guess would be worse than reporting none. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
d1faa04771
commit
3e71e9e88b
@@ -131,11 +131,47 @@ def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Map
|
||||
f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |"
|
||||
)
|
||||
lines += ["", _credit_note(models), ""]
|
||||
cutoff = _cutoff_note(models, families)
|
||||
if cutoff:
|
||||
lines += [cutoff, ""]
|
||||
if lever is not None:
|
||||
lines += [_lever_note(lever), ""]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _cutoff_note(models: Mapping[str, Baseline], families: tuple[str, ...]) -> str:
|
||||
"""The post-cutoff slice, its cutoff and the undated count (Req 10.5).
|
||||
|
||||
A model that may have read the fix cannot be measured on it the same way as one that cannot have, so the rows
|
||||
fixed after the detector's training cutoff are reported apart. Rows the corpus records no date for are in
|
||||
neither slice, and their number is here rather than folded silently into the whole."""
|
||||
rows = [
|
||||
(model, family, baseline.metrics[family])
|
||||
for model, baseline in sorted(models.items())
|
||||
for family in families
|
||||
if family in baseline.metrics and str(baseline.metrics[family].get("cutoff") or "")
|
||||
]
|
||||
if not rows:
|
||||
return "No detector cutoff is configured, so no post-cutoff slice is reported; every figure above is over the whole corpus."
|
||||
cutoff = str(rows[0][2].get("cutoff"))
|
||||
lines = [
|
||||
f"Post-cutoff slice, fixed after {cutoff} and therefore beyond the detector's training data. "
|
||||
"A row counts only when its earliest possible date is after the cutoff, so a year-precision date can "
|
||||
"under-claim this slice but never over-claim it.",
|
||||
"",
|
||||
"| model | family | post-cutoff scorable | post-cutoff recall | post-cutoff Youden | undated |",
|
||||
"|---|---|---|---|---|---|",
|
||||
]
|
||||
for model, family, row in rows:
|
||||
block = row.get("post_cutoff")
|
||||
block = block if isinstance(block, Mapping) else {}
|
||||
lines.append(
|
||||
f"| {model} | {family} | {block.get('scorable', 0)} | {_num(block.get('recall'))} "
|
||||
f"| {_num(block.get('youden'))} | {row.get('undated', 0)} |"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _credit_note(models: Mapping[str, Baseline]) -> str:
|
||||
hierarchical = any(bool(row.get("hierarchical_credit")) for baseline in models.values() for row in baseline.metrics.values())
|
||||
if hierarchical:
|
||||
|
||||
Reference in New Issue
Block a user