Open_Ultra_SAST/src/openultrasast/learning/publish.py

"""Every published number, regenerated from the artifacts that produced it (learning-harness, Req 11.1, 11.2, 4.5, 5.3).

A number in a roadmap that nobody can regenerate is a claim, not a measurement. This writes them all
from one entry point, out of the artifacts committed beside them, and puts the denominator next to each:
how many pairs could be scored, why the rest could not, which taxonomy version scored them, the noise
the family shows against itself, and what the run cost per pair it got right.

Running it twice without new artifacts changes nothing, and the roadmap section is replaced in place
rather than appended, so the file cannot accumulate stale numbers beside fresh ones.
"""

from __future__ import annotations

import json
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path

BEGIN = "<!-- learning-harness:begin -->"
END = "<!-- learning-harness:end -->"
COMMAND = "uv run ousast learning publish"


@dataclass(frozen=True)
class Baseline:
    """One model's round-zero artifact, read back."""

    model: str
    metrics: dict[str, Mapping[str, object]]
    floors: dict[str, Mapping[str, object]]
    cost_usd: float
    k_runs: int


@dataclass(frozen=True)
class PublishReport:
    command: str
    table: str
    families: tuple[str, ...] = ()
    models: tuple[str, ...] = ()
    artifacts: tuple[Path, ...] = field(default_factory=tuple)


def publish(*, learning_dir: Path, measurements_dir: Path, roadmap: Path, date: str | None = None) -> PublishReport:
    """Regenerate the family numbers, write the artifact, and replace the roadmap section in place."""
    stamp = date or datetime.now(UTC).strftime("%Y-%m-%d")
    models = _baselines(learning_dir)
    families = tuple(sorted({family for baseline in models.values() for family in baseline.metrics}))
    lever = _lever(measurements_dir)
    table = _table(models, families, lever)
    artifacts: list[Path] = []
    if models:
        measurements_dir.mkdir(parents=True, exist_ok=True)
        artifact = measurements_dir / f"{stamp}-learning-families.json"
        artifact.write_text(
            json.dumps(
                {
                    "generated": stamp,
                    "command": COMMAND,
                    "models": {name: dict(baseline.metrics) for name, baseline in sorted(models.items())},
                    "floors": {name: dict(baseline.floors) for name, baseline in sorted(models.items())},
                },
                indent=2,
                sort_keys=True,
            )
            + "\n",
            encoding="utf-8",
        )
        artifacts.append(artifact)
    _replace_section(roadmap, table)
    return PublishReport(command=COMMAND, table=table, families=families, models=tuple(sorted(models)), artifacts=tuple(artifacts))


def _baselines(learning_dir: Path) -> dict[str, Baseline]:
    root = learning_dir / "baseline"
    found: dict[str, Baseline] = {}
    for directory in sorted(path for path in root.iterdir() if path.is_dir()) if root.is_dir() else []:
        report = directory / "report.json"
        if not report.is_file():
            continue
        payload = json.loads(report.read_text(encoding="utf-8"))
        model = str(payload.get("model", directory.name))
        found[model] = Baseline(
            model=model,
            metrics={str(name): block for name, block in (payload.get("metrics") or {}).items() if isinstance(block, Mapping)},
            floors={str(name): block for name, block in (payload.get("floors") or {}).items() if isinstance(block, Mapping)},
            cost_usd=float(payload.get("cost_usd", 0.0) or 0.0),
            k_runs=int(payload.get("k_runs", 0) or 0),
        )
    return found


def _lever(measurements_dir: Path) -> Mapping[str, object] | None:
    path = measurements_dir / "2026-09-06-mechanism-lever-split.json"
    if not path.is_file():
        return None
    payload = json.loads(path.read_text(encoding="utf-8"))
    return payload if isinstance(payload, dict) else None


def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Mapping[str, object] | None) -> str:
    if not models:
        return (
            f"Generated by `{COMMAND}`.\n\nNo baseline has been recorded yet, so there is nothing to publish. "
            "Run round zero first; a number without the run that produced it is a claim, not a measurement.\n"
        )
    lines = [
        f"Generated by `{COMMAND}` from the artifacts committed beside these numbers.",
        "",
        "| model | family | scorable | unscorable | recall | silence | Youden | at fixed FPR | noise floor | cost per correct pair |",
        "|---|---|---|---|---|---|---|---|---|---|",
    ]
    for model, baseline in sorted(models.items()):
        for family in families:
            row = baseline.metrics.get(family)
            if row is None:
                continue
            outcomes = row.get("outcomes")
            raw_correct = outcomes.get("pair_correct") if isinstance(outcomes, Mapping) else None
            correct = int(raw_correct) if isinstance(raw_correct, int | float) else 0
            floor = baseline.floors.get(family)
            raw_budget = floor.get("budget_flips") if floor is not None else None
            budget = int(raw_budget) if isinstance(raw_budget, int | float) else 0
            cost = baseline.cost_usd
            lines.append(
                f"| {model} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) "
                f"| {_reasons(row)} | {_num(row.get('recall'))} | {_num(row.get('silence'))} | {_num(row.get('youden'))} "
                f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |"
            )
    lines += ["", _credit_note(models), ""]
    if lever is not None:
        lines += [_lever_note(lever), ""]
    return "\n".join(lines)


def _credit_note(models: Mapping[str, Baseline]) -> str:
    hierarchical = any(bool(row.get("hierarchical_credit")) for baseline in models.values() for row in baseline.metrics.values())
    if hierarchical:
        return "A parent-child answer earned partial credit in these figures."
    return "No family in this taxonomy declares a parent, so hierarchical partial credit could not apply to any figure above."


def _lever_note(lever: Mapping[str, object]) -> str:
    before = lever.get("before")
    after = lever.get("after")
    before_map = before if isinstance(before, Mapping) else {}
    after_map = after if isinstance(after, Mapping) else {}
    pairs = lever.get("holdout_pairs", 0)
    return (
        f"Mechanism lever on the {lever.get('slice', '')} holdout: detected {after_map.get('detected')} of {pairs} at Youden "
        f"{after_map.get('youden')} once only train-split pairs teach, against detected {before_map.get('detected')} at Youden "
        f"{before_map.get('youden')} when every pair taught. The difference is what holdout pairs stopped teaching."
    )


def _reasons(row: Mapping[str, object]) -> str:
    reasons = row.get("unscorable") or {}
    if not isinstance(reasons, Mapping) or not reasons:
        return "none"
    return ", ".join(f"{name}: {count}" for name, count in sorted(reasons.items()))


def _num(value: object) -> str:
    return f"{float(value):.3f}" if isinstance(value, int | float) else "-"


def _cost(cost: float, correct: int) -> str:
    return f"{cost / correct:.2f}" if cost and correct else "-"


def _replace_section(roadmap: Path, table: str) -> None:
    body = f"{BEGIN}\n\n## Classified detector numbers\n\n{table.rstrip()}\n\n{END}\n"
    text = roadmap.read_text(encoding="utf-8") if roadmap.is_file() else ""
    if BEGIN in text and END in text:
        head, _, rest = text.partition(BEGIN)
        _, _, tail = rest.partition(END)
        roadmap.write_text(head + body + tail.lstrip("\n"), encoding="utf-8")
        return
    roadmap.parent.mkdir(parents=True, exist_ok=True)
    roadmap.write_text((text.rstrip("\n") + "\n\n" if text else "") + body, encoding="utf-8")


__all__ = ["BEGIN", "COMMAND", "END", "Baseline", "PublishReport", "publish"]