mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
feat(learning-harness): publish every number from the artifacts that produced it
Task 3.8. A number in a roadmap that nobody can regenerate is a claim, not a measurement. One entry point now writes them all from the committed artifacts and puts the denominator beside each: how many pairs could be scored, why the rest could not, which taxonomy version scored them, the noise the family shows against itself, and what the run cost per pair it got right. Every table says whether hierarchical partial credit could apply, which today it cannot. The roadmap section is replaced in place between markers rather than appended, so the file cannot accumulate a stale number beside a fresh one, and running the command twice without new artifacts changes nothing. With no baseline recorded it says so instead of printing an empty table. The corrected mechanism-lever number travels with the rest: detected five of seventeen at a negative Youden once only train-split pairs teach, against seven when every pair taught, with the difference named as what holdout pairs stopped teaching. Round zero now records what it spent so the cost column has a source. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
b9552518b9
commit
b8ad42c57e
@@ -188,3 +188,20 @@ def test_an_unscorable_pair_is_listed_and_never_run(tmp_path: Path) -> None:
|
||||
assert report.metrics["injection"]["scorable"] == 0
|
||||
assert report.metrics["injection"]["unscorable"] == {"identical_twin": 1}
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_round_zero_records_what_it_spent(tmp_path: Path) -> None:
|
||||
"""Publishing puts a cost beside every number, so the baseline has to know what it cost."""
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
report = run_baseline(
|
||||
[_case(tmp_path, "a")],
|
||||
taxonomy=load_families(),
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan=_steady,
|
||||
model="m",
|
||||
out_dir=tmp_path / "out",
|
||||
spent_usd=lambda: 1.25,
|
||||
)
|
||||
assert report.cost_usd == 1.25
|
||||
assert json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text())["cost_usd"] == 1.25
|
||||
|
||||
Reference in New Issue
Block a user