From 6d9850578eb52bfc4f7614c5fb483f0bc4335999 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:23:36 +0200 Subject: [PATCH] learn: dry-run estimate from measured cached-response cost; docs: exp-004 result; tidy instrument check The estimator priced all input at the full rate (\$5.58 estimated vs \$1.24 spent); it now uses the family mean billed cost of cached responses when usage is recorded. Implemented with Codex. Co-Authored-By: Claude Opus 5.5 --- .../instrument_check.py | 83 +++++++++++-------- docs/decision-engine.md | 48 +++++++++-- src/openultrasast/learn/fill.py | 20 +++-- tests/test_learn_fill.py | 53 ++++++++++-- 4 files changed, 152 insertions(+), 52 deletions(-) diff --git a/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py b/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py index a347f5b..cd9e857 100644 --- a/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py +++ b/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py @@ -1,36 +1,53 @@ -"""Instrument check for exp-004: are the scores real, and how clean is the pooled top of the ranking?""" -import json, collections +"""Instrument check for exp-004: are the scores real, and how clean is the pooled top of the ranking? + +Reads the run's arm_outcome rows from the memory store (OUSAST_MEMORY) and the frozen units file; makes no model call. +usage: python benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py (from the repository root) +""" + +from __future__ import annotations + +import collections +import json from pathlib import Path + from openultrasast.config import load_dotenv -load_dotenv(Path(".env")) -from openultrasast.plane.memory import open_store from openultrasast.learn import experiments as ex -store = open_store(None) -rows = [r.row for r in store.rows(repo=ex.experiment_repo("exp-004-pooled-block-gate"), kind="arm_outcome")] -units = {json.loads(l)["unit"]: json.loads(l) for l in open("plane/experiments/exp-004-pooled-block-gate.units.jsonl")} -print("arm_outcome rows", len(rows), "units file", len(units)) -r0 = rows[0]; print("row keys", sorted(r0)[:30]) -print("decision example", json.dumps(r0.get("decision"))[:300]) -def score(r): - d = r.get("decision") or {} - return r.get("s"), r.get("verdict", "unsure" if r.get("s") is None else "") -fam = collections.defaultdict(list) -for r in rows: - u = units.get(r.get("unit")) - if not u: continue - p, v = score(r) - fam[u["family"]].append((p, v, u["label"], u.get("pair"))) -for f, xs in sorted(fam.items()): - ps = [x for x in xs if x[0] is not None] - distinct = len({round(x[0], 4) for x in ps}) - unsure = sum(1 for x in xs if x[1] == "unsure") - pairs = collections.defaultdict(dict) - for p, v, y, pr in xs: - if pr and p is not None: pairs[pr][y] = p - full = [d for d in pairs.values() if 0 in d and 1 in d] - wins = sum(1 for d in full if d[1] > d[0]); ties = sum(1 for d in full if d[1] == d[0]) - top = sorted([x for x in ps if x[3]], key=lambda x: -x[0]) - first_neg = next((i for i, x in enumerate(top) if x[2] == 0), len(top)) - print(f"{f}: n={len(xs)} distinct_scores={distinct} unsure={unsure} pairs={len(full)} within_pair_win={wins} tie={ties} pos_above_first_neg={first_neg}") -allp = sorted([x for xs in fam.values() for x in xs if x[0] is not None and x[3]], key=lambda x: -x[0]) -print("pooled raw top-30 labels:", "".join(str(x[2]) for x in allp[:30])) +from openultrasast.plane.memory import open_store + +EXPERIMENT = "exp-004-pooled-block-gate" +UNITS = Path(f"plane/experiments/{EXPERIMENT}.units.jsonl") + + +def main() -> None: + load_dotenv(Path(".env")) + rows = [r.row for r in open_store(None).rows(repo=ex.experiment_repo(EXPERIMENT), kind="arm_outcome")] + with UNITS.open() as handle: + units = {u["unit"]: u for u in (json.loads(line) for line in handle)} + print("arm_outcome rows", len(rows), "units file", len(units)) + by_family: dict[str, list[tuple[float | None, int, str | None, str]]] = collections.defaultdict(list) + for row in rows: + unit = units.get(row.get("unit")) + if unit: + by_family[unit["family"]].append((row.get("s"), unit["label"], unit.get("pair"), str(row.get("verdict") or ""))) + for family, xs in sorted(by_family.items()): + scored = [x for x in xs if x[0] is not None] + pairs: dict[str, dict[int, float]] = collections.defaultdict(dict) + for score, label, pair, _ in scored: + if pair: + pairs[pair][label] = score + full = [p for p in pairs.values() if 0 in p and 1 in p] + wins = sum(1 for p in full if p[1] > p[0]) + ties = sum(1 for p in full if p[1] == p[0]) + top = sorted((x for x in scored if x[2]), key=lambda x: -(x[0] or 0.0)) + first_negative = next((i for i, x in enumerate(top) if x[1] == 0), len(top)) + print( + f"{family}: n={len(xs)} distinct_scores={len({round(x[0] or 0.0, 4) for x in scored})} " + f"unsure={sum(1 for x in xs if x[3] == 'unsure')} pairs={len(full)} within_pair_win={wins} tie={ties} " + f"pos_above_first_neg={first_negative}" + ) + pooled = sorted((x for xs in by_family.values() for x in xs if x[0] is not None and x[2]), key=lambda x: -(x[0] or 0.0)) + print("pooled raw top-30 labels:", "".join(str(x[1]) for x in pooled[:30])) + + +if __name__ == "__main__": + main() diff --git a/docs/decision-engine.md b/docs/decision-engine.md index 3882e5f..fc2bdf8 100644 --- a/docs/decision-engine.md +++ b/docs/decision-engine.md @@ -150,7 +150,7 @@ Four experiments are registered: 0.875 against 0.906 and an interval spanning zero (`benchmarks/experiments/exp-002-retrieval-ensemble/result.json`). - exp-003 compared two BLOCK rules on the same scores. It was **inconclusive**. Neither rule flags anything. -- exp-004 tests one pooled BLOCK threshold with a per-family floor. It is registered and has not run. +- exp-004 tested one pooled BLOCK threshold with a per-family floor. It was **inconclusive**. ### exp-003: precision-bound blocking (2026-10-03) @@ -216,8 +216,44 @@ Calibration holds only for untrusted destination. There arm A still finds no BLOCK point that reaches the bound. Record: `benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json`. +### exp-004: pooled block gate (2026-10-03) + +The experiment tested one pooled BLOCK threshold across families. +Each family needed at least 10 clean flags to block. +Each family also had its own quiet threshold. +The run evaluated 501 units across 220 repository groups. +Of these, 390 units were paired. +The cache fill cost $1.238223 for 2,073 calls. +The replay cost $0.0 and made 0 client calls. + +The verdict was **inconclusive**. +No fold found a `t_block` threshold. +The instrument check confirmed real scores. +The pooled top 30 contained 27 positives. +The first negative appeared at rank 12. +The bound needs 73 clean flags without an error. + +A within-pair win means the vulnerable side scored above its fixed counterpart. + +| Family | Within-pair wins | Pairs | +| --- | --- | --- | +| access control | 22 | 28 | +| deserialization | 18 | 25 | +| injection | 30 | 53 | +| output encoding | 18 | 28 | +| path | 25 | 29 | +| untrusted destination | 18 | 32 | + +Precision at the top of the ranking now limits blocking. +Data volume alone cannot resolve this. +The next levers are graph evidence in the prompt and the combiner. + +Records: `benchmarks/experiments/exp-004-pooled-block-gate/record.json` and `benchmarks/experiments/exp-004-pooled-block-gate/result.json`. + ### exp-004: pooled block gate with a family floor (registered, not run) +This registration describes the setup before the run reported above. + The maintainer amended the BLOCK rule on 2026-10-03 (requirements, Req 6.4). Precision is now shown across all families together, not per family. Each family's score is first mapped by its own Platt fit, on training folds. @@ -230,8 +266,8 @@ No held-out label moves a threshold. - **Primary metric:** the pooled held-out Wilson lower bound of flagged precision, paired units, before the floor. - **Decision:** reject below point precision 0.95; adopt if the bound reaches 0.95 and a family passes the floor. -- **Units:** frozen at run time, from the memory after the negatives harvest. -- **Cost:** the run replays the cache. The new memory needs a paid cache fill first. +- **Units:** frozen from the memory after the negatives harvest. +- **Cost:** the run replayed the cache after a paid cache fill. Manifest: `plane/experiments/exp-004-pooled-block-gate.yaml`. @@ -267,8 +303,10 @@ which writes nothing. Exit codes are 0 for completion or dry-run, 1 for a provid failure, 2 for invalid setup and 3 for budget exhaustion. Resume a partial fill with a new invocation and budget; that budget covers only the new invocation, including smoke. -Dry-run prices each missing prompt using the program's input-token estimate and the -family's mean cached completion-token usage, falling back to **300 output tokens**. +Dry-run uses each family's mean billed cached response cost when recorded usage exists. +Model prices account for cache-hit input, cache-miss input, and output tokens. +Without cached usage, it uses the program's input-token estimate and **300 output tokens**. +Each family's `estimate_method` reports `measured` or `estimated`. It reports potential retries separately as `conditional_retries`: an unseen answer may need one parse retry, whose necessity cannot be known without calling the model. Known invalid cached answers include their retry requests in the inventory. diff --git a/src/openultrasast/learn/fill.py b/src/openultrasast/learn/fill.py index 9c8e771..36d9139 100644 --- a/src/openultrasast/learn/fill.py +++ b/src/openultrasast/learn/fill.py @@ -1,7 +1,7 @@ """Fill rule-experiment responses without writing scores or changing the frozen population. -Estimates use Prompt.estimated_tokens and the family's mean cached completion usage, -or 300 output tokens when no usage is available. Unknown answers may require one retry; +Estimates use the family's mean billed cached response cost when usage is available. +Otherwise they use Prompt.estimated_tokens and 300 output tokens. Unknown answers may require one retry; dry-run counts those separately as conditional requests, never fabricating an answer. """ @@ -13,7 +13,7 @@ from dataclasses import dataclass from typing import Any from ..model.endpoint import DeepSeekChatClient -from ..plane.budget import BudgetExhausted, MeteredClient, prices_from +from ..plane.budget import BudgetExhausted, MeteredClient, cost_of, prices_from from ..plane.memory import MemoryStore from ..tool_hunter import ChatClient, ChatResponse from . import experiments as ex @@ -143,11 +143,12 @@ def fill_rule( initially_cached = {key for cells in initial.values() for key in cells if caller._cached(key) is not None} families: dict[str, Any] = {} for family, cells in initial.items(): - outputs = [ - float((entry.get("usage") or {})["completion_tokens"]) + usages = [ + usage for key in cells - if (entry := caller._cached(key)) is not None and "completion_tokens" in (entry.get("usage") or {}) + if (entry := caller._cached(key)) is not None and isinstance(usage := entry.get("usage"), Mapping) and usage ] + outputs = [float(usage["completion_tokens"]) for usage in usages if "completion_tokens" in usage] mean_output = sum(outputs) / len(outputs) if outputs else DEFAULT_OUTPUT_TOKENS missing = [req for key, req in cells.items() if key not in initially_cached] families[family] = { @@ -157,9 +158,10 @@ def fill_rule( "missing": len(missing), "filled": 0, "still_missing": len(missing), - "estimated_usd": sum( - (r.prompt.estimated_tokens * prices.input_per_m + mean_output * prices.output_per_m) / 1_000_000 for r in missing - ), + "estimate_method": "measured" if usages else "estimated", + "estimated_usd": len(missing) * sum(cost_of(usage, prices) for usage in usages) / len(usages) + if usages + else sum((r.prompt.estimated_tokens * prices.input_per_m + mean_output * prices.output_per_m) / 1_000_000 for r in missing), "estimated_output_tokens": mean_output, "conditional_retries": sum(not r.sample.endswith(":retry") for r in missing), } diff --git a/tests/test_learn_fill.py b/tests/test_learn_fill.py index 1d7fd7b..139fdd7 100644 --- a/tests/test_learn_fill.py +++ b/tests/test_learn_fill.py @@ -14,7 +14,7 @@ from test_learn_rules import prepared, write_rule_manifest from openultrasast.cli import main from openultrasast.learn import experiments as ex from openultrasast.learn.fill import fill_rule -from openultrasast.learn.program import Candidate, Program +from openultrasast.learn.program import Candidate, Program, Prompt from openultrasast.learn.retrieve import Target from openultrasast.plane.memory import FileStore @@ -70,6 +70,49 @@ def test_fill_only_missing_requests_then_replay_is_complete(tmp_path: Path) -> N assert meter.calls == 0 and meter.usd == 0 +@pytest.mark.parametrize("explicit_miss", [False, True]) +def test_dry_run_uses_mean_billed_cached_usage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, explicit_miss: bool) -> None: + store, manifest, units = prepared(tmp_path) + assert seed_one(store, manifest, units[0]) == 2 + paths = sorted((tmp_path / "memory" / "responses").rglob("*")) + responses = [p for p in paths if p.is_file()] + assert len(responses) == 2 + for i, path in enumerate(responses, 1): + entry = json.loads(path.read_bytes()) + entry["usage"] = {"prompt_cache_hit_tokens": 9000 * i, "completion_tokens": 100 * i} + entry["usage"]["prompt_cache_miss_tokens" if explicit_miss else "prompt_tokens"] = (1000 if explicit_miss else 10000) * i + path.write_text(json.dumps(entry)) + monkeypatch.setattr(Prompt, "estimated_tokens", property(lambda self: 1_000_000)) + before = snapshot(tmp_path) + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True) + family = report["families"][units[0].family] + mean_cost = 1.5 * (9000 * 0.014 + 1000 * 0.44 + 100 * 1.32) / 1_000_000 + assert family["missing"] > 0 and family["estimate_method"] == "measured" + assert family["estimated_usd"] == pytest.approx(family["missing"] * mean_cost) + other = next(f for name, f in report["families"].items() if name != units[0].family) + assert other["estimate_method"] == "estimated" + assert report["client_calls"] == 0 and snapshot(tmp_path) == before + + +@pytest.mark.parametrize("usage", [None, {}]) +def test_dry_run_falls_back_without_cached_usage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, usage: dict[str, int] | None) -> None: + store, manifest, units = prepared(tmp_path) + assert seed_one(store, manifest, units[0]) == 2 + for path in (tmp_path / "memory" / "responses").rglob("*"): + if path.is_file(): + entry = json.loads(path.read_bytes()) + entry.pop("usage") + if usage is not None: + entry["usage"] = usage + path.write_text(json.dumps(entry)) + monkeypatch.setattr(Prompt, "estimated_tokens", property(lambda self: 1000)) + report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True) + for family in report["families"].values(): + assert family["estimate_method"] == "estimated" + assert family["estimated_output_tokens"] == 300 + assert family["estimated_usd"] == pytest.approx(family["missing"] * (1000 * 0.44 + 300 * 1.32) / 1_000_000) + + class FixedCostChat(ScriptedChat): def complete(self, **kwargs: Any) -> Any: response = super().complete(**kwargs) @@ -181,6 +224,7 @@ def test_cli_dry_run_needs_no_endpoint_and_does_not_write_out( assert code == 0 and not out.exists() and snapshot(tmp_path) == before report = json.loads(capsys.readouterr().out) assert report["client_calls"] == 0 and report["status"] == "dry_run" + assert all(f["estimate_method"] == "estimated" for f in report["families"].values()) def test_growing_call_costs_stay_inside_the_initial_reserve(tmp_path: Path) -> None: @@ -200,20 +244,19 @@ def test_growing_call_costs_stay_inside_the_initial_reserve(tmp_path: Path) -> N assert total(report, "filled") == len(chat.calls) and total(report, "still_missing") > 0 -def test_dry_run_prices_prompt_tokens_and_family_cached_output_mean(tmp_path: Path) -> None: +def test_dry_run_prices_prompt_tokens_with_empty_cache(tmp_path: Path) -> None: from openultrasast.learn.fill import DEFAULT_OUTPUT_TOKENS store, manifest, units = prepared(tmp_path) - seed_one(store, manifest, units[0]) report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True) folds = {f.name: f for f in ex.program_folds(EXAMPLES, manifest)[0]} index = {e.id: e for e in EXAMPLES} expected = dict.fromkeys(manifest.families, 0.0) - for unit in units[1:]: + for unit in units: example = index[unit.unit] program = Program(ex.arm_spec(manifest, manifest.arms["A"], unit.family, store), EXAMPLES, excerpt_text) prompt, _ = program.prepare(Candidate(Target.of(example), CODE[example.excerpt_sha], "python"), folds[unit.fold], seed=0) - output_tokens = 60 if unit.family == units[0].family else DEFAULT_OUTPUT_TOKENS + output_tokens = DEFAULT_OUTPUT_TOKENS expected[unit.family] += ( program.spec.k * (prompt.estimated_tokens * PRICES["input_per_m"] + output_tokens * PRICES["output_per_m"]) / 1_000_000 )