diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 3611978..58d77cf 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -1272,6 +1272,10 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: if resolved is None: print("learning: learning_endpoint_unavailable; scoring what can be scored without a detector") client = resolved[0] if resolved else None + # The client meters itself; a run that never reads the meter publishes a cost of zero and can never trip its + # own cap (Req 8.3, 9.8). + meter = getattr(client, "cost_usd", None) + spent: Callable[[], float] | None = meter if callable(meter) else None configs = load_family_configs(configs_dir, taxonomy) # type: ignore[arg-type] def factory(family_config: object) -> Callable[[Path], list[StaticFinding]]: @@ -1323,7 +1327,9 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: # `--slice` already chose the rows; without this the baseline's own default filters them out again and # a run over the memory-safety slice reports nothing at all (Req 8.4). slices=(args.slice,) if args.slice != "all" else DEFAULT_SLICES, + spent_usd=spent, ) + print(f"learning baseline cost: ${report.cost_usd:.4f}") print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}") return 0 journal = LearningJournal(out / "journal.jsonl") @@ -1342,11 +1348,12 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd, minibatch=config.learning.minibatch, k_runs=k_runs, + spent_usd=spent, # Every other family the corpus actually carries, so a change that reaches beyond its own directory is # measured rather than assumed impossible (Req 9.5). sweep_families=tuple(sorted({_label(case, taxonomy) for case in cases} & set(configs) - {args.family})), ) - print(f"learning round {record.round} {record.family}: {record.outcome} ({record.reason})") + print(f"learning round {record.round} {record.family}: {record.outcome} ({record.reason}) cost ${record.cost_usd:.4f}") return 0 diff --git a/tests/test_learning_cli.py b/tests/test_learning_cli.py index f20bb48..ef9658b 100644 --- a/tests/test_learning_cli.py +++ b/tests/test_learning_cli.py @@ -253,3 +253,48 @@ def test_the_pairs_command_scores_k_runs_per_side(workspace: Path, monkeypatch: monkeypatch.setenv("OPENULTRASAST_HUNTER_MODEL", "scripted") assert main(["pairs", "--catalog", str(workspace / "pairs.toml"), "--hunter", "--hunter-model", "scripted", "--k-runs", "4"]) == 0 assert calls.count("vuln") == 8 and calls.count("fixed") == 8 # two pairs, four runs per side + + +def test_a_baseline_records_what_the_endpoint_actually_spent(workspace: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Req 8.3: cost per pair is published beside every number, so the run has to read it off the client. + + The client meters itself; nothing passed the meter to the baseline, so every published cost would have been + 0.0 — and the round's cost cap, which reads the same meter, could never have fired either (Req 9.8).""" + from openultrasast.learning.endpoint import ChatEndpoint + from openultrasast.tool_hunter import ChatResponse + + class Metered: + def __init__(self) -> None: + self.calls = 0 + + def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def] + del model, messages, tools, options + self.calls += 1 + return ChatResponse(content="[]") + + def cost_usd(self) -> float: + return 0.25 * self.calls + + client = Metered() + endpoint = ChatEndpoint(provider="scripted", base_url="", thinking=False) + monkeypatch.setattr("openultrasast.learning.endpoint.resolve_chat_endpoint", lambda *a, **k: (client, endpoint)) + assert ( + main( + [ + "learning", + "baseline", + "--catalog", + str(workspace / "pairs.toml"), + "--out", + str(workspace / "learning"), + "--k-runs", + "3", + "--model", + "metered", + ] + ) + == 0 + ) + report = json.loads((workspace / "learning" / "baseline" / "metered" / "report.json").read_text()) + assert client.calls > 0 + assert report["cost_usd"] == pytest.approx(0.25 * client.calls)