diff --git a/src/openultrasast/learning/publish.py b/src/openultrasast/learning/publish.py new file mode 100644 index 0000000..aa0f0d9 --- /dev/null +++ b/src/openultrasast/learning/publish.py @@ -0,0 +1,184 @@ +"""Every published number, regenerated from the artifacts that produced it (learning-harness, Req 11.1, 11.2, 4.5, 5.3). + +A number in a roadmap that nobody can regenerate is a claim, not a measurement. This writes them all +from one entry point, out of the artifacts committed beside them, and puts the denominator next to each: +how many pairs could be scored, why the rest could not, which taxonomy version scored them, the noise +the family shows against itself, and what the run cost per pair it got right. + +Running it twice without new artifacts changes nothing, and the roadmap section is replaced in place +rather than appended, so the file cannot accumulate stale numbers beside fresh ones. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path + +BEGIN = "" +END = "" +COMMAND = "uv run ousast learning publish" + + +@dataclass(frozen=True) +class Baseline: + """One model's round-zero artifact, read back.""" + + model: str + metrics: dict[str, Mapping[str, object]] + floors: dict[str, Mapping[str, object]] + cost_usd: float + k_runs: int + + +@dataclass(frozen=True) +class PublishReport: + command: str + table: str + families: tuple[str, ...] = () + models: tuple[str, ...] = () + artifacts: tuple[Path, ...] = field(default_factory=tuple) + + +def publish(*, learning_dir: Path, measurements_dir: Path, roadmap: Path, date: str | None = None) -> PublishReport: + """Regenerate the family numbers, write the artifact, and replace the roadmap section in place.""" + stamp = date or datetime.now(UTC).strftime("%Y-%m-%d") + models = _baselines(learning_dir) + families = tuple(sorted({family for baseline in models.values() for family in baseline.metrics})) + lever = _lever(measurements_dir) + table = _table(models, families, lever) + artifacts: list[Path] = [] + if models: + measurements_dir.mkdir(parents=True, exist_ok=True) + artifact = measurements_dir / f"{stamp}-learning-families.json" + artifact.write_text( + json.dumps( + { + "generated": stamp, + "command": COMMAND, + "models": {name: dict(baseline.metrics) for name, baseline in sorted(models.items())}, + "floors": {name: dict(baseline.floors) for name, baseline in sorted(models.items())}, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + artifacts.append(artifact) + _replace_section(roadmap, table) + return PublishReport(command=COMMAND, table=table, families=families, models=tuple(sorted(models)), artifacts=tuple(artifacts)) + + +def _baselines(learning_dir: Path) -> dict[str, Baseline]: + root = learning_dir / "baseline" + found: dict[str, Baseline] = {} + for directory in sorted(path for path in root.iterdir() if path.is_dir()) if root.is_dir() else []: + report = directory / "report.json" + if not report.is_file(): + continue + payload = json.loads(report.read_text(encoding="utf-8")) + model = str(payload.get("model", directory.name)) + found[model] = Baseline( + model=model, + metrics={str(name): block for name, block in (payload.get("metrics") or {}).items() if isinstance(block, Mapping)}, + floors={str(name): block for name, block in (payload.get("floors") or {}).items() if isinstance(block, Mapping)}, + cost_usd=float(payload.get("cost_usd", 0.0) or 0.0), + k_runs=int(payload.get("k_runs", 0) or 0), + ) + return found + + +def _lever(measurements_dir: Path) -> Mapping[str, object] | None: + path = measurements_dir / "2026-09-06-mechanism-lever-split.json" + if not path.is_file(): + return None + payload = json.loads(path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else None + + +def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Mapping[str, object] | None) -> str: + if not models: + return ( + f"Generated by `{COMMAND}`.\n\nNo baseline has been recorded yet, so there is nothing to publish. " + "Run round zero first; a number without the run that produced it is a claim, not a measurement.\n" + ) + lines = [ + f"Generated by `{COMMAND}` from the artifacts committed beside these numbers.", + "", + "| model | family | scorable | unscorable | recall | silence | Youden | at fixed FPR | noise floor | cost per correct pair |", + "|---|---|---|---|---|---|---|---|---|---|", + ] + for model, baseline in sorted(models.items()): + for family in families: + row = baseline.metrics.get(family) + if row is None: + continue + outcomes = row.get("outcomes") + raw_correct = outcomes.get("pair_correct") if isinstance(outcomes, Mapping) else None + correct = int(raw_correct) if isinstance(raw_correct, int | float) else 0 + floor = baseline.floors.get(family) + raw_budget = floor.get("budget_flips") if floor is not None else None + budget = int(raw_budget) if isinstance(raw_budget, int | float) else 0 + cost = baseline.cost_usd + lines.append( + f"| {model} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) " + f"| {_reasons(row)} | {_num(row.get('recall'))} | {_num(row.get('silence'))} | {_num(row.get('youden'))} " + f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |" + ) + lines += ["", _credit_note(models), ""] + if lever is not None: + lines += [_lever_note(lever), ""] + return "\n".join(lines) + + +def _credit_note(models: Mapping[str, Baseline]) -> str: + hierarchical = any(bool(row.get("hierarchical_credit")) for baseline in models.values() for row in baseline.metrics.values()) + if hierarchical: + return "A parent-child answer earned partial credit in these figures." + return "No family in this taxonomy declares a parent, so hierarchical partial credit could not apply to any figure above." + + +def _lever_note(lever: Mapping[str, object]) -> str: + before = lever.get("before") + after = lever.get("after") + before_map = before if isinstance(before, Mapping) else {} + after_map = after if isinstance(after, Mapping) else {} + pairs = lever.get("holdout_pairs", 0) + return ( + f"Mechanism lever on the {lever.get('slice', '')} holdout: detected {after_map.get('detected')} of {pairs} at Youden " + f"{after_map.get('youden')} once only train-split pairs teach, against detected {before_map.get('detected')} at Youden " + f"{before_map.get('youden')} when every pair taught. The difference is what holdout pairs stopped teaching." + ) + + +def _reasons(row: Mapping[str, object]) -> str: + reasons = row.get("unscorable") or {} + if not isinstance(reasons, Mapping) or not reasons: + return "none" + return ", ".join(f"{name}: {count}" for name, count in sorted(reasons.items())) + + +def _num(value: object) -> str: + return f"{float(value):.3f}" if isinstance(value, int | float) else "-" + + +def _cost(cost: float, correct: int) -> str: + return f"{cost / correct:.2f}" if cost and correct else "-" + + +def _replace_section(roadmap: Path, table: str) -> None: + body = f"{BEGIN}\n\n## Classified detector numbers\n\n{table.rstrip()}\n\n{END}\n" + text = roadmap.read_text(encoding="utf-8") if roadmap.is_file() else "" + if BEGIN in text and END in text: + head, _, rest = text.partition(BEGIN) + _, _, tail = rest.partition(END) + roadmap.write_text(head + body + tail.lstrip("\n"), encoding="utf-8") + return + roadmap.parent.mkdir(parents=True, exist_ok=True) + roadmap.write_text((text.rstrip("\n") + "\n\n" if text else "") + body, encoding="utf-8") + + +__all__ = ["BEGIN", "COMMAND", "END", "Baseline", "PublishReport", "publish"] diff --git a/src/openultrasast/learning/rounds.py b/src/openultrasast/learning/rounds.py index 59e2a59..563a549 100644 --- a/src/openultrasast/learning/rounds.py +++ b/src/openultrasast/learning/rounds.py @@ -69,6 +69,7 @@ class BaselineReport: floors: dict[str, NoiseFloor] = field(default_factory=dict) metrics: dict[str, dict[str, object]] = field(default_factory=dict) scores: tuple[PairFamilyScore, ...] = () + cost_usd: float = 0.0 def to_dict(self) -> dict[str, object]: return { @@ -76,6 +77,7 @@ class BaselineReport: "k_runs": self.k_runs, "taxonomy_version": self.taxonomy_version, "slices": list(self.slices), + "cost_usd": self.cost_usd, "floors": {name: floor.to_dict() for name, floor in sorted(self.floors.items())}, "metrics": {name: dict(block) for name, block in sorted(self.metrics.items())}, } @@ -92,6 +94,7 @@ def run_baseline( slices: Sequence[str] = DEFAULT_SLICES, k_runs: int = DEFAULT_K_RUNS, prompt: str | None = None, + spent_usd: Callable[[], float] | None = None, ) -> BaselineReport: """Clone one detector into every family, measure each family against itself, and write the artifacts.""" from ..tool_hunter import _SYSTEM_PROMPT @@ -110,6 +113,7 @@ def run_baseline( floors=floors, metrics={name: block.to_dict() for name, block in metrics.items()}, scores=tuple(scores), + cost_usd=spent_usd() if spent_usd is not None else 0.0, ) _write(out_dir, report) return report diff --git a/tests/test_learning_publish.py b/tests/test_learning_publish.py new file mode 100644 index 0000000..281a908 --- /dev/null +++ b/tests/test_learning_publish.py @@ -0,0 +1,158 @@ +"""learning-harness task 3.8: publishing every number from its committed artifacts (offline).""" + +from __future__ import annotations + +import json +from pathlib import Path + +ROADMAP = "# Roadmap\n\n## Overview\n\nSomething about the project.\n\n## Constraints\n\nCore dependencies stay empty.\n" + + +def _baseline(out: Path, model: str, *, recall: float = 1.0, cost: float = 0.5) -> None: + directory = out / "baseline" / model + directory.mkdir(parents=True, exist_ok=True) + (directory / "report.json").write_text( + json.dumps( + { + "model": model, + "k_runs": 5, + "taxonomy_version": "1", + "slices": ["vibe-py"], + "cost_usd": cost, + "floors": {"injection": {"family": "injection", "budget_flips": 1, "negative_flip_rate": 0.1, "k_runs": 5, "gates": True}}, + "metrics": { + "injection": { + "family": "injection", + "taxonomy_version": "1", + "scorable": 10, + "unscorable": {"identical_twin": 2}, + "outcomes": {"pair_correct": round(recall * 10)}, # keep the fixture internally consistent + "recall": recall, + "silence": 1.0, + "youden": recall, + "directional_bias": 0.0, + "leak_rate": 0.0, + "fixed_fpr_recall": recall, + "hierarchical_credit": False, + } + }, + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + + +def test_publishing_twice_without_new_artifacts_is_byte_identical(tmp_path: Path) -> None: + from openultrasast.learning.publish import publish + + out = tmp_path / "learning" + _baseline(out, "deepseek-v4-flash") + roadmap = tmp_path / "roadmap.md" + roadmap.write_text(ROADMAP) + measurements = tmp_path / "measurements" + first = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) + text = roadmap.read_text() + second = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) + assert roadmap.read_text() == text + assert first.table == second.table and first.families == second.families + + +def test_every_number_carries_its_denominator_floor_and_cost(tmp_path: Path) -> None: + from openultrasast.learning.publish import publish + + out = tmp_path / "learning" + _baseline(out, "deepseek-v4-flash", recall=0.9, cost=1.8) + roadmap = tmp_path / "roadmap.md" + roadmap.write_text(ROADMAP) + report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) + table = report.table + assert "| injection |" in table + assert "10" in table and "identical_twin: 2" in table # scorable and why the rest are not + assert "taxonomy 1" in table and "budget 1" in table + assert "0.20" in table # cost per pair-correct: 1.8 over 9 + assert "hierarchical partial credit could not apply" in table + assert report.command and report.command in roadmap.read_text() + + +def test_a_second_model_appears_beside_the_first(tmp_path: Path) -> None: + from openultrasast.learning.publish import publish + + out = tmp_path / "learning" + _baseline(out, "deepseek-v4-flash", recall=0.9) + _baseline(out, "deepseek-v4-pro", recall=1.0) + roadmap = tmp_path / "roadmap.md" + roadmap.write_text(ROADMAP) + report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) + assert "deepseek-v4-flash" in report.table and "deepseek-v4-pro" in report.table + assert set(report.families) == {"injection"} + + +def test_the_corrected_lever_number_replaces_the_earlier_one(tmp_path: Path) -> None: + from openultrasast.learning.publish import publish + + out = tmp_path / "learning" + _baseline(out, "deepseek-v4-flash") + measurements = tmp_path / "m" + measurements.mkdir() + (measurements / "2026-09-06-mechanism-lever-split.json").write_text( + json.dumps( + { + "slice": "vibe-py", + "holdout_pairs": 17, + "before": {"detected": 7, "youden": 0.0588}, + "after": {"detected": 5, "youden": -0.0588}, + } + ) + ) + roadmap = tmp_path / "roadmap.md" + roadmap.write_text(ROADMAP + "\nThe lever recovered 7 of 17 holdout pairs at Youden +0.059.\n") + report = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) + text = roadmap.read_text() + assert "-0.0588" in report.table or "-0.059" in report.table + assert "5 of 17" in report.table or "detected 5" in report.table + assert "holdout pairs stopped teaching" in report.table.lower() or "train split only" in report.table.lower() + assert text.count("learning-harness:begin") == 1 + + +def test_the_roadmap_section_is_replaced_not_appended_again(tmp_path: Path) -> None: + from openultrasast.learning.publish import publish + + out = tmp_path / "learning" + _baseline(out, "deepseek-v4-flash", recall=0.5) + roadmap = tmp_path / "roadmap.md" + roadmap.write_text(ROADMAP) + publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) + _baseline(out, "deepseek-v4-flash", recall=0.9) + publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) + text = roadmap.read_text() + assert text.count("learning-harness:begin") == 1 and text.count("learning-harness:end") == 1 + assert "## Overview" in text and "## Constraints" in text # nothing else in the roadmap is disturbed + assert "0.5" not in text.split("learning-harness:begin")[1] # the stale number is gone + + +def test_the_raw_artifacts_are_written_where_the_numbers_are_read_from(tmp_path: Path) -> None: + from openultrasast.learning.publish import publish + + out = tmp_path / "learning" + _baseline(out, "deepseek-v4-flash") + measurements = tmp_path / "m" + roadmap = tmp_path / "roadmap.md" + roadmap.write_text(ROADMAP) + publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) + written = sorted(path.name for path in measurements.iterdir()) + assert any(name.endswith("-learning-families.json") for name in written) + payload = json.loads((measurements / next(name for name in written if name.endswith("-learning-families.json"))).read_text()) + assert payload["models"]["deepseek-v4-flash"]["injection"]["scorable"] == 10 + assert payload["command"] + + +def test_publishing_with_no_baseline_says_so_rather_than_inventing_a_table(tmp_path: Path) -> None: + from openultrasast.learning.publish import publish + + roadmap = tmp_path / "roadmap.md" + roadmap.write_text(ROADMAP) + report = publish(learning_dir=tmp_path / "empty", measurements_dir=tmp_path / "m", roadmap=roadmap) + assert report.families == () and "no baseline" in report.table.lower() + assert "learning-harness:begin" in roadmap.read_text() diff --git a/tests/test_learning_rounds.py b/tests/test_learning_rounds.py index eb8dd12..910e036 100644 --- a/tests/test_learning_rounds.py +++ b/tests/test_learning_rounds.py @@ -188,3 +188,20 @@ def test_an_unscorable_pair_is_listed_and_never_run(tmp_path: Path) -> None: assert report.metrics["injection"]["scorable"] == 0 assert report.metrics["injection"]["unscorable"] == {"identical_twin": 1} assert calls == [] + + +def test_round_zero_records_what_it_spent(tmp_path: Path) -> None: + """Publishing puts a cost beside every number, so the baseline has to know what it cost.""" + from openultrasast.learning.rounds import run_baseline + + report = run_baseline( + [_case(tmp_path, "a")], + taxonomy=load_families(), + configs_dir=tmp_path / "configs", + scan=_steady, + model="m", + out_dir=tmp_path / "out", + spent_usd=lambda: 1.25, + ) + assert report.cost_usd == 1.25 + assert json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text())["cost_usd"] == 1.25