mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(learning-harness): publish every number from the artifacts that produced it
Task 3.8. A number in a roadmap that nobody can regenerate is a claim, not a measurement. One entry point now writes them all from the committed artifacts and puts the denominator beside each: how many pairs could be scored, why the rest could not, which taxonomy version scored them, the noise the family shows against itself, and what the run cost per pair it got right. Every table says whether hierarchical partial credit could apply, which today it cannot. The roadmap section is replaced in place between markers rather than appended, so the file cannot accumulate a stale number beside a fresh one, and running the command twice without new artifacts changes nothing. With no baseline recorded it says so instead of printing an empty table. The corrected mechanism-lever number travels with the rest: detected five of seventeen at a negative Youden once only train-split pairs teach, against seven when every pair taught, with the difference named as what holdout pairs stopped teaching. Round zero now records what it spent so the cost column has a source. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5
Claude Opus 5
parent
b9552518b9
commit
b8ad42c57e
@@ -0,0 +1,184 @@
|
||||
"""Every published number, regenerated from the artifacts that produced it (learning-harness, Req 11.1, 11.2, 4.5, 5.3).
|
||||
|
||||
A number in a roadmap that nobody can regenerate is a claim, not a measurement. This writes them all
|
||||
from one entry point, out of the artifacts committed beside them, and puts the denominator next to each:
|
||||
how many pairs could be scored, why the rest could not, which taxonomy version scored them, the noise
|
||||
the family shows against itself, and what the run cost per pair it got right.
|
||||
|
||||
Running it twice without new artifacts changes nothing, and the roadmap section is replaced in place
|
||||
rather than appended, so the file cannot accumulate stale numbers beside fresh ones.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
BEGIN = "<!-- learning-harness:begin -->"
|
||||
END = "<!-- learning-harness:end -->"
|
||||
COMMAND = "uv run ousast learning publish"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Baseline:
|
||||
"""One model's round-zero artifact, read back."""
|
||||
|
||||
model: str
|
||||
metrics: dict[str, Mapping[str, object]]
|
||||
floors: dict[str, Mapping[str, object]]
|
||||
cost_usd: float
|
||||
k_runs: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublishReport:
|
||||
command: str
|
||||
table: str
|
||||
families: tuple[str, ...] = ()
|
||||
models: tuple[str, ...] = ()
|
||||
artifacts: tuple[Path, ...] = field(default_factory=tuple)
|
||||
|
||||
|
||||
def publish(*, learning_dir: Path, measurements_dir: Path, roadmap: Path, date: str | None = None) -> PublishReport:
|
||||
"""Regenerate the family numbers, write the artifact, and replace the roadmap section in place."""
|
||||
stamp = date or datetime.now(UTC).strftime("%Y-%m-%d")
|
||||
models = _baselines(learning_dir)
|
||||
families = tuple(sorted({family for baseline in models.values() for family in baseline.metrics}))
|
||||
lever = _lever(measurements_dir)
|
||||
table = _table(models, families, lever)
|
||||
artifacts: list[Path] = []
|
||||
if models:
|
||||
measurements_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact = measurements_dir / f"{stamp}-learning-families.json"
|
||||
artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"generated": stamp,
|
||||
"command": COMMAND,
|
||||
"models": {name: dict(baseline.metrics) for name, baseline in sorted(models.items())},
|
||||
"floors": {name: dict(baseline.floors) for name, baseline in sorted(models.items())},
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
artifacts.append(artifact)
|
||||
_replace_section(roadmap, table)
|
||||
return PublishReport(command=COMMAND, table=table, families=families, models=tuple(sorted(models)), artifacts=tuple(artifacts))
|
||||
|
||||
|
||||
def _baselines(learning_dir: Path) -> dict[str, Baseline]:
|
||||
root = learning_dir / "baseline"
|
||||
found: dict[str, Baseline] = {}
|
||||
for directory in sorted(path for path in root.iterdir() if path.is_dir()) if root.is_dir() else []:
|
||||
report = directory / "report.json"
|
||||
if not report.is_file():
|
||||
continue
|
||||
payload = json.loads(report.read_text(encoding="utf-8"))
|
||||
model = str(payload.get("model", directory.name))
|
||||
found[model] = Baseline(
|
||||
model=model,
|
||||
metrics={str(name): block for name, block in (payload.get("metrics") or {}).items() if isinstance(block, Mapping)},
|
||||
floors={str(name): block for name, block in (payload.get("floors") or {}).items() if isinstance(block, Mapping)},
|
||||
cost_usd=float(payload.get("cost_usd", 0.0) or 0.0),
|
||||
k_runs=int(payload.get("k_runs", 0) or 0),
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
def _lever(measurements_dir: Path) -> Mapping[str, object] | None:
|
||||
path = measurements_dir / "2026-09-06-mechanism-lever-split.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Mapping[str, object] | None) -> str:
|
||||
if not models:
|
||||
return (
|
||||
f"Generated by `{COMMAND}`.\n\nNo baseline has been recorded yet, so there is nothing to publish. "
|
||||
"Run round zero first; a number without the run that produced it is a claim, not a measurement.\n"
|
||||
)
|
||||
lines = [
|
||||
f"Generated by `{COMMAND}` from the artifacts committed beside these numbers.",
|
||||
"",
|
||||
"| model | family | scorable | unscorable | recall | silence | Youden | at fixed FPR | noise floor | cost per correct pair |",
|
||||
"|---|---|---|---|---|---|---|---|---|---|",
|
||||
]
|
||||
for model, baseline in sorted(models.items()):
|
||||
for family in families:
|
||||
row = baseline.metrics.get(family)
|
||||
if row is None:
|
||||
continue
|
||||
outcomes = row.get("outcomes")
|
||||
raw_correct = outcomes.get("pair_correct") if isinstance(outcomes, Mapping) else None
|
||||
correct = int(raw_correct) if isinstance(raw_correct, int | float) else 0
|
||||
floor = baseline.floors.get(family)
|
||||
raw_budget = floor.get("budget_flips") if floor is not None else None
|
||||
budget = int(raw_budget) if isinstance(raw_budget, int | float) else 0
|
||||
cost = baseline.cost_usd
|
||||
lines.append(
|
||||
f"| {model} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) "
|
||||
f"| {_reasons(row)} | {_num(row.get('recall'))} | {_num(row.get('silence'))} | {_num(row.get('youden'))} "
|
||||
f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |"
|
||||
)
|
||||
lines += ["", _credit_note(models), ""]
|
||||
if lever is not None:
|
||||
lines += [_lever_note(lever), ""]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _credit_note(models: Mapping[str, Baseline]) -> str:
|
||||
hierarchical = any(bool(row.get("hierarchical_credit")) for baseline in models.values() for row in baseline.metrics.values())
|
||||
if hierarchical:
|
||||
return "A parent-child answer earned partial credit in these figures."
|
||||
return "No family in this taxonomy declares a parent, so hierarchical partial credit could not apply to any figure above."
|
||||
|
||||
|
||||
def _lever_note(lever: Mapping[str, object]) -> str:
|
||||
before = lever.get("before")
|
||||
after = lever.get("after")
|
||||
before_map = before if isinstance(before, Mapping) else {}
|
||||
after_map = after if isinstance(after, Mapping) else {}
|
||||
pairs = lever.get("holdout_pairs", 0)
|
||||
return (
|
||||
f"Mechanism lever on the {lever.get('slice', '')} holdout: detected {after_map.get('detected')} of {pairs} at Youden "
|
||||
f"{after_map.get('youden')} once only train-split pairs teach, against detected {before_map.get('detected')} at Youden "
|
||||
f"{before_map.get('youden')} when every pair taught. The difference is what holdout pairs stopped teaching."
|
||||
)
|
||||
|
||||
|
||||
def _reasons(row: Mapping[str, object]) -> str:
|
||||
reasons = row.get("unscorable") or {}
|
||||
if not isinstance(reasons, Mapping) or not reasons:
|
||||
return "none"
|
||||
return ", ".join(f"{name}: {count}" for name, count in sorted(reasons.items()))
|
||||
|
||||
|
||||
def _num(value: object) -> str:
|
||||
return f"{float(value):.3f}" if isinstance(value, int | float) else "-"
|
||||
|
||||
|
||||
def _cost(cost: float, correct: int) -> str:
|
||||
return f"{cost / correct:.2f}" if cost and correct else "-"
|
||||
|
||||
|
||||
def _replace_section(roadmap: Path, table: str) -> None:
|
||||
body = f"{BEGIN}\n\n## Classified detector numbers\n\n{table.rstrip()}\n\n{END}\n"
|
||||
text = roadmap.read_text(encoding="utf-8") if roadmap.is_file() else ""
|
||||
if BEGIN in text and END in text:
|
||||
head, _, rest = text.partition(BEGIN)
|
||||
_, _, tail = rest.partition(END)
|
||||
roadmap.write_text(head + body + tail.lstrip("\n"), encoding="utf-8")
|
||||
return
|
||||
roadmap.parent.mkdir(parents=True, exist_ok=True)
|
||||
roadmap.write_text((text.rstrip("\n") + "\n\n" if text else "") + body, encoding="utf-8")
|
||||
|
||||
|
||||
__all__ = ["BEGIN", "COMMAND", "END", "Baseline", "PublishReport", "publish"]
|
||||
@@ -69,6 +69,7 @@ class BaselineReport:
|
||||
floors: dict[str, NoiseFloor] = field(default_factory=dict)
|
||||
metrics: dict[str, dict[str, object]] = field(default_factory=dict)
|
||||
scores: tuple[PairFamilyScore, ...] = ()
|
||||
cost_usd: float = 0.0
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
@@ -76,6 +77,7 @@ class BaselineReport:
|
||||
"k_runs": self.k_runs,
|
||||
"taxonomy_version": self.taxonomy_version,
|
||||
"slices": list(self.slices),
|
||||
"cost_usd": self.cost_usd,
|
||||
"floors": {name: floor.to_dict() for name, floor in sorted(self.floors.items())},
|
||||
"metrics": {name: dict(block) for name, block in sorted(self.metrics.items())},
|
||||
}
|
||||
@@ -92,6 +94,7 @@ def run_baseline(
|
||||
slices: Sequence[str] = DEFAULT_SLICES,
|
||||
k_runs: int = DEFAULT_K_RUNS,
|
||||
prompt: str | None = None,
|
||||
spent_usd: Callable[[], float] | None = None,
|
||||
) -> BaselineReport:
|
||||
"""Clone one detector into every family, measure each family against itself, and write the artifacts."""
|
||||
from ..tool_hunter import _SYSTEM_PROMPT
|
||||
@@ -110,6 +113,7 @@ def run_baseline(
|
||||
floors=floors,
|
||||
metrics={name: block.to_dict() for name, block in metrics.items()},
|
||||
scores=tuple(scores),
|
||||
cost_usd=spent_usd() if spent_usd is not None else 0.0,
|
||||
)
|
||||
_write(out_dir, report)
|
||||
return report
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""learning-harness task 3.8: publishing every number from its committed artifacts (offline)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROADMAP = "# Roadmap\n\n## Overview\n\nSomething about the project.\n\n## Constraints\n\nCore dependencies stay empty.\n"
|
||||
|
||||
|
||||
def _baseline(out: Path, model: str, *, recall: float = 1.0, cost: float = 0.5) -> None:
|
||||
directory = out / "baseline" / model
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
(directory / "report.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model": model,
|
||||
"k_runs": 5,
|
||||
"taxonomy_version": "1",
|
||||
"slices": ["vibe-py"],
|
||||
"cost_usd": cost,
|
||||
"floors": {"injection": {"family": "injection", "budget_flips": 1, "negative_flip_rate": 0.1, "k_runs": 5, "gates": True}},
|
||||
"metrics": {
|
||||
"injection": {
|
||||
"family": "injection",
|
||||
"taxonomy_version": "1",
|
||||
"scorable": 10,
|
||||
"unscorable": {"identical_twin": 2},
|
||||
"outcomes": {"pair_correct": round(recall * 10)}, # keep the fixture internally consistent
|
||||
"recall": recall,
|
||||
"silence": 1.0,
|
||||
"youden": recall,
|
||||
"directional_bias": 0.0,
|
||||
"leak_rate": 0.0,
|
||||
"fixed_fpr_recall": recall,
|
||||
"hierarchical_credit": False,
|
||||
}
|
||||
},
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
def test_publishing_twice_without_new_artifacts_is_byte_identical(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash")
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
measurements = tmp_path / "measurements"
|
||||
first = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
|
||||
text = roadmap.read_text()
|
||||
second = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
|
||||
assert roadmap.read_text() == text
|
||||
assert first.table == second.table and first.families == second.families
|
||||
|
||||
|
||||
def test_every_number_carries_its_denominator_floor_and_cost(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash", recall=0.9, cost=1.8)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
table = report.table
|
||||
assert "| injection |" in table
|
||||
assert "10" in table and "identical_twin: 2" in table # scorable and why the rest are not
|
||||
assert "taxonomy 1" in table and "budget 1" in table
|
||||
assert "0.20" in table # cost per pair-correct: 1.8 over 9
|
||||
assert "hierarchical partial credit could not apply" in table
|
||||
assert report.command and report.command in roadmap.read_text()
|
||||
|
||||
|
||||
def test_a_second_model_appears_beside_the_first(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash", recall=0.9)
|
||||
_baseline(out, "deepseek-v4-pro", recall=1.0)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
assert "deepseek-v4-flash" in report.table and "deepseek-v4-pro" in report.table
|
||||
assert set(report.families) == {"injection"}
|
||||
|
||||
|
||||
def test_the_corrected_lever_number_replaces_the_earlier_one(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash")
|
||||
measurements = tmp_path / "m"
|
||||
measurements.mkdir()
|
||||
(measurements / "2026-09-06-mechanism-lever-split.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"slice": "vibe-py",
|
||||
"holdout_pairs": 17,
|
||||
"before": {"detected": 7, "youden": 0.0588},
|
||||
"after": {"detected": 5, "youden": -0.0588},
|
||||
}
|
||||
)
|
||||
)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP + "\nThe lever recovered 7 of 17 holdout pairs at Youden +0.059.\n")
|
||||
report = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
|
||||
text = roadmap.read_text()
|
||||
assert "-0.0588" in report.table or "-0.059" in report.table
|
||||
assert "5 of 17" in report.table or "detected 5" in report.table
|
||||
assert "holdout pairs stopped teaching" in report.table.lower() or "train split only" in report.table.lower()
|
||||
assert text.count("learning-harness:begin") == 1
|
||||
|
||||
|
||||
def test_the_roadmap_section_is_replaced_not_appended_again(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash", recall=0.5)
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
_baseline(out, "deepseek-v4-flash", recall=0.9)
|
||||
publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
text = roadmap.read_text()
|
||||
assert text.count("learning-harness:begin") == 1 and text.count("learning-harness:end") == 1
|
||||
assert "## Overview" in text and "## Constraints" in text # nothing else in the roadmap is disturbed
|
||||
assert "0.5" not in text.split("learning-harness:begin")[1] # the stale number is gone
|
||||
|
||||
|
||||
def test_the_raw_artifacts_are_written_where_the_numbers_are_read_from(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
out = tmp_path / "learning"
|
||||
_baseline(out, "deepseek-v4-flash")
|
||||
measurements = tmp_path / "m"
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap)
|
||||
written = sorted(path.name for path in measurements.iterdir())
|
||||
assert any(name.endswith("-learning-families.json") for name in written)
|
||||
payload = json.loads((measurements / next(name for name in written if name.endswith("-learning-families.json"))).read_text())
|
||||
assert payload["models"]["deepseek-v4-flash"]["injection"]["scorable"] == 10
|
||||
assert payload["command"]
|
||||
|
||||
|
||||
def test_publishing_with_no_baseline_says_so_rather_than_inventing_a_table(tmp_path: Path) -> None:
|
||||
from openultrasast.learning.publish import publish
|
||||
|
||||
roadmap = tmp_path / "roadmap.md"
|
||||
roadmap.write_text(ROADMAP)
|
||||
report = publish(learning_dir=tmp_path / "empty", measurements_dir=tmp_path / "m", roadmap=roadmap)
|
||||
assert report.families == () and "no baseline" in report.table.lower()
|
||||
assert "learning-harness:begin" in roadmap.read_text()
|
||||
@@ -188,3 +188,20 @@ def test_an_unscorable_pair_is_listed_and_never_run(tmp_path: Path) -> None:
|
||||
assert report.metrics["injection"]["scorable"] == 0
|
||||
assert report.metrics["injection"]["unscorable"] == {"identical_twin": 1}
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_round_zero_records_what_it_spent(tmp_path: Path) -> None:
|
||||
"""Publishing puts a cost beside every number, so the baseline has to know what it cost."""
|
||||
from openultrasast.learning.rounds import run_baseline
|
||||
|
||||
report = run_baseline(
|
||||
[_case(tmp_path, "a")],
|
||||
taxonomy=load_families(),
|
||||
configs_dir=tmp_path / "configs",
|
||||
scan=_steady,
|
||||
model="m",
|
||||
out_dir=tmp_path / "out",
|
||||
spent_usd=lambda: 1.25,
|
||||
)
|
||||
assert report.cost_usd == 1.25
|
||||
assert json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text())["cost_usd"] == 1.25
|
||||
|
||||
Reference in New Issue
Block a user