learn: dry-run estimate from measured cached-response cost; docs: exp-004 result; tidy instrument check

The estimator priced all input at the full rate (\$5.58 estimated vs \$1.24
spent); it now uses the family mean billed cost of cached responses when usage
is recorded. Implemented with Codex.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py

@@ -1,36 +1,53 @@
-"""Instrument check for exp-004: are the scores real, and how clean is the pooled top of the ranking?"""
-import json, collections
+"""Instrument check for exp-004: are the scores real, and how clean is the pooled top of the ranking?
+
+Reads the run's arm_outcome rows from the memory store (OUSAST_MEMORY) and the frozen units file; makes no model call.
+usage: python benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py   (from the repository root)
+"""
+
+from __future__ import annotations
+
+import collections
+import json
 from pathlib import Path
+
 from openultrasast.config import load_dotenv
-load_dotenv(Path(".env"))
-from openultrasast.plane.memory import open_store
 from openultrasast.learn import experiments as ex
-store = open_store(None)
-rows = [r.row for r in store.rows(repo=ex.experiment_repo("exp-004-pooled-block-gate"), kind="arm_outcome")]
-units = {json.loads(l)["unit"]: json.loads(l) for l in open("plane/experiments/exp-004-pooled-block-gate.units.jsonl")}
+from openultrasast.plane.memory import open_store
+
+EXPERIMENT = "exp-004-pooled-block-gate"
+UNITS = Path(f"plane/experiments/{EXPERIMENT}.units.jsonl")
+
+
+def main() -> None:
+    load_dotenv(Path(".env"))
+    rows = [r.row for r in open_store(None).rows(repo=ex.experiment_repo(EXPERIMENT), kind="arm_outcome")]
+    with UNITS.open() as handle:
+        units = {u["unit"]: u for u in (json.loads(line) for line in handle)}
     print("arm_outcome rows", len(rows), "units file", len(units))
-r0 = rows[0]; print("row keys", sorted(r0)[:30])
-print("decision example", json.dumps(r0.get("decision"))[:300])
-def score(r):
-    d = r.get("decision") or {}
-    return r.get("s"), r.get("verdict", "unsure" if r.get("s") is None else "")
-fam = collections.defaultdict(list)
-for r in rows:
-    u = units.get(r.get("unit"))
-    if not u: continue
-    p, v = score(r)
-    fam[u["family"]].append((p, v, u["label"], u.get("pair")))
-for f, xs in sorted(fam.items()):
-    ps = [x for x in xs if x[0] is not None]
-    distinct = len({round(x[0], 4) for x in ps})
-    unsure = sum(1 for x in xs if x[1] == "unsure")
-    pairs = collections.defaultdict(dict)
-    for p, v, y, pr in xs:
-        if pr and p is not None: pairs[pr][y] = p
-    full = [d for d in pairs.values() if 0 in d and 1 in d]
-    wins = sum(1 for d in full if d[1] > d[0]); ties = sum(1 for d in full if d[1] == d[0])
-    top = sorted([x for x in ps if x[3]], key=lambda x: -x[0])
-    first_neg = next((i for i, x in enumerate(top) if x[2] == 0), len(top))
-    print(f"{f}: n={len(xs)} distinct_scores={distinct} unsure={unsure} pairs={len(full)} within_pair_win={wins} tie={ties} pos_above_first_neg={first_neg}")
-allp = sorted([x for xs in fam.values() for x in xs if x[0] is not None and x[3]], key=lambda x: -x[0])
-print("pooled raw top-30 labels:", "".join(str(x[2]) for x in allp[:30]))
+    by_family: dict[str, list[tuple[float | None, int, str | None, str]]] = collections.defaultdict(list)
+    for row in rows:
+        unit = units.get(row.get("unit"))
+        if unit:
+            by_family[unit["family"]].append((row.get("s"), unit["label"], unit.get("pair"), str(row.get("verdict") or "")))
+    for family, xs in sorted(by_family.items()):
+        scored = [x for x in xs if x[0] is not None]
+        pairs: dict[str, dict[int, float]] = collections.defaultdict(dict)
+        for score, label, pair, _ in scored:
+            if pair:
+                pairs[pair][label] = score
+        full = [p for p in pairs.values() if 0 in p and 1 in p]
+        wins = sum(1 for p in full if p[1] > p[0])
+        ties = sum(1 for p in full if p[1] == p[0])
+        top = sorted((x for x in scored if x[2]), key=lambda x: -(x[0] or 0.0))
+        first_negative = next((i for i, x in enumerate(top) if x[1] == 0), len(top))
+        print(
+            f"{family}: n={len(xs)} distinct_scores={len({round(x[0] or 0.0, 4) for x in scored})} "
+            f"unsure={sum(1 for x in xs if x[3] == 'unsure')} pairs={len(full)} within_pair_win={wins} tie={ties} "
+            f"pos_above_first_neg={first_negative}"
+        )
+    pooled = sorted((x for xs in by_family.values() for x in xs if x[0] is not None and x[2]), key=lambda x: -(x[0] or 0.0))
+    print("pooled raw top-30 labels:", "".join(str(x[1]) for x in pooled[:30]))
+
+
+if __name__ == "__main__":
+    main()

docs/decision-engine.md

@@ -150,7 +150,7 @@ Four experiments are registered:
   0.875 against 0.906 and an interval spanning zero
   (`benchmarks/experiments/exp-002-retrieval-ensemble/result.json`).
 - exp-003 compared two BLOCK rules on the same scores. It was **inconclusive**. Neither rule flags anything.
-- exp-004 tests one pooled BLOCK threshold with a per-family floor. It is registered and has not run.
+- exp-004 tested one pooled BLOCK threshold with a per-family floor. It was **inconclusive**.
 
 ### exp-003: precision-bound blocking (2026-10-03)
 
@@ -216,8 +216,44 @@ Calibration holds only for untrusted destination.
 There arm A still finds no BLOCK point that reaches the bound.
 Record: `benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json`.
 
+### exp-004: pooled block gate (2026-10-03)
+
+The experiment tested one pooled BLOCK threshold across families.
+Each family needed at least 10 clean flags to block.
+Each family also had its own quiet threshold.
+The run evaluated 501 units across 220 repository groups.
+Of these, 390 units were paired.
+The cache fill cost $1.238223 for 2,073 calls.
+The replay cost $0.0 and made 0 client calls.
+
+The verdict was **inconclusive**.
+No fold found a `t_block` threshold.
+The instrument check confirmed real scores.
+The pooled top 30 contained 27 positives.
+The first negative appeared at rank 12.
+The bound needs 73 clean flags without an error.
+
+A within-pair win means the vulnerable side scored above its fixed counterpart.
+
+| Family | Within-pair wins | Pairs |
+| --- | --- | --- |
+| access control | 22 | 28 |
+| deserialization | 18 | 25 |
+| injection | 30 | 53 |
+| output encoding | 18 | 28 |
+| path | 25 | 29 |
+| untrusted destination | 18 | 32 |
+
+Precision at the top of the ranking now limits blocking.
+Data volume alone cannot resolve this.
+The next levers are graph evidence in the prompt and the combiner.
+
+Records: `benchmarks/experiments/exp-004-pooled-block-gate/record.json` and `benchmarks/experiments/exp-004-pooled-block-gate/result.json`.
+
 ### exp-004: pooled block gate with a family floor (registered, not run)
 
+This registration describes the setup before the run reported above.
+
 The maintainer amended the BLOCK rule on 2026-10-03 (requirements, Req 6.4).
 Precision is now shown across all families together, not per family.
 Each family's score is first mapped by its own Platt fit, on training folds.
@@ -230,8 +266,8 @@ No held-out label moves a threshold.
 
 - **Primary metric:** the pooled held-out Wilson lower bound of flagged precision, paired units, before the floor.
 - **Decision:** reject below point precision 0.95; adopt if the bound reaches 0.95 and a family passes the floor.
-- **Units:** frozen at run time, from the memory after the negatives harvest.
-- **Cost:** the run replays the cache. The new memory needs a paid cache fill first.
+- **Units:** frozen from the memory after the negatives harvest.
+- **Cost:** the run replayed the cache after a paid cache fill.
 
 Manifest: `plane/experiments/exp-004-pooled-block-gate.yaml`.
 
@@ -267,8 +303,10 @@ which writes nothing. Exit codes are 0 for completion or dry-run, 1 for a provid
 failure, 2 for invalid setup and 3 for budget exhaustion. Resume a partial fill with a
 new invocation and budget; that budget covers only the new invocation, including smoke.
 
-Dry-run prices each missing prompt using the program's input-token estimate and the
-family's mean cached completion-token usage, falling back to **300 output tokens**.
+Dry-run uses each family's mean billed cached response cost when recorded usage exists.
+Model prices account for cache-hit input, cache-miss input, and output tokens.
+Without cached usage, it uses the program's input-token estimate and **300 output tokens**.
+Each family's `estimate_method` reports `measured` or `estimated`.
 It reports potential retries separately as `conditional_retries`: an unseen answer
 may need one parse retry, whose necessity cannot be known without calling the model.
 Known invalid cached answers include their retry requests in the inventory.

src/openultrasast/learn/fill.py

@@ -1,7 +1,7 @@
 """Fill rule-experiment responses without writing scores or changing the frozen population.
 
-Estimates use Prompt.estimated_tokens and the family's mean cached completion usage,
-or 300 output tokens when no usage is available. Unknown answers may require one retry;
+Estimates use the family's mean billed cached response cost when usage is available.
+Otherwise they use Prompt.estimated_tokens and 300 output tokens. Unknown answers may require one retry;
 dry-run counts those separately as conditional requests, never fabricating an answer.
 """
 
@@ -13,7 +13,7 @@ from dataclasses import dataclass
 from typing import Any
 
 from ..model.endpoint import DeepSeekChatClient
-from ..plane.budget import BudgetExhausted, MeteredClient, prices_from
+from ..plane.budget import BudgetExhausted, MeteredClient, cost_of, prices_from
 from ..plane.memory import MemoryStore
 from ..tool_hunter import ChatClient, ChatResponse
 from . import experiments as ex
@@ -143,11 +143,12 @@ def fill_rule(
     initially_cached = {key for cells in initial.values() for key in cells if caller._cached(key) is not None}
     families: dict[str, Any] = {}
     for family, cells in initial.items():
-        outputs = [
-            float((entry.get("usage") or {})["completion_tokens"])
+        usages = [
+            usage
             for key in cells
-            if (entry := caller._cached(key)) is not None and "completion_tokens" in (entry.get("usage") or {})
+            if (entry := caller._cached(key)) is not None and isinstance(usage := entry.get("usage"), Mapping) and usage
         ]
+        outputs = [float(usage["completion_tokens"]) for usage in usages if "completion_tokens" in usage]
         mean_output = sum(outputs) / len(outputs) if outputs else DEFAULT_OUTPUT_TOKENS
         missing = [req for key, req in cells.items() if key not in initially_cached]
         families[family] = {
@@ -157,9 +158,10 @@ def fill_rule(
             "missing": len(missing),
             "filled": 0,
             "still_missing": len(missing),
-            "estimated_usd": sum(
-                (r.prompt.estimated_tokens * prices.input_per_m + mean_output * prices.output_per_m) / 1_000_000 for r in missing
-            ),
+            "estimate_method": "measured" if usages else "estimated",
+            "estimated_usd": len(missing) * sum(cost_of(usage, prices) for usage in usages) / len(usages)
+            if usages
+            else sum((r.prompt.estimated_tokens * prices.input_per_m + mean_output * prices.output_per_m) / 1_000_000 for r in missing),
             "estimated_output_tokens": mean_output,
             "conditional_retries": sum(not r.sample.endswith(":retry") for r in missing),
         }

tests/test_learn_fill.py

@@ -14,7 +14,7 @@ from test_learn_rules import prepared, write_rule_manifest
 from openultrasast.cli import main
 from openultrasast.learn import experiments as ex
 from openultrasast.learn.fill import fill_rule
-from openultrasast.learn.program import Candidate, Program
+from openultrasast.learn.program import Candidate, Program, Prompt
 from openultrasast.learn.retrieve import Target
 from openultrasast.plane.memory import FileStore
 
@@ -70,6 +70,49 @@ def test_fill_only_missing_requests_then_replay_is_complete(tmp_path: Path) -> N
     assert meter.calls == 0 and meter.usd == 0
 
 
+@pytest.mark.parametrize("explicit_miss", [False, True])
+def test_dry_run_uses_mean_billed_cached_usage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, explicit_miss: bool) -> None:
+    store, manifest, units = prepared(tmp_path)
+    assert seed_one(store, manifest, units[0]) == 2
+    paths = sorted((tmp_path / "memory" / "responses").rglob("*"))
+    responses = [p for p in paths if p.is_file()]
+    assert len(responses) == 2
+    for i, path in enumerate(responses, 1):
+        entry = json.loads(path.read_bytes())
+        entry["usage"] = {"prompt_cache_hit_tokens": 9000 * i, "completion_tokens": 100 * i}
+        entry["usage"]["prompt_cache_miss_tokens" if explicit_miss else "prompt_tokens"] = (1000 if explicit_miss else 10000) * i
+        path.write_text(json.dumps(entry))
+    monkeypatch.setattr(Prompt, "estimated_tokens", property(lambda self: 1_000_000))
+    before = snapshot(tmp_path)
+    report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True)
+    family = report["families"][units[0].family]
+    mean_cost = 1.5 * (9000 * 0.014 + 1000 * 0.44 + 100 * 1.32) / 1_000_000
+    assert family["missing"] > 0 and family["estimate_method"] == "measured"
+    assert family["estimated_usd"] == pytest.approx(family["missing"] * mean_cost)
+    other = next(f for name, f in report["families"].items() if name != units[0].family)
+    assert other["estimate_method"] == "estimated"
+    assert report["client_calls"] == 0 and snapshot(tmp_path) == before
+
+
+@pytest.mark.parametrize("usage", [None, {}])
+def test_dry_run_falls_back_without_cached_usage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, usage: dict[str, int] | None) -> None:
+    store, manifest, units = prepared(tmp_path)
+    assert seed_one(store, manifest, units[0]) == 2
+    for path in (tmp_path / "memory" / "responses").rglob("*"):
+        if path.is_file():
+            entry = json.loads(path.read_bytes())
+            entry.pop("usage")
+            if usage is not None:
+                entry["usage"] = usage
+            path.write_text(json.dumps(entry))
+    monkeypatch.setattr(Prompt, "estimated_tokens", property(lambda self: 1000))
+    report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True)
+    for family in report["families"].values():
+        assert family["estimate_method"] == "estimated"
+        assert family["estimated_output_tokens"] == 300
+        assert family["estimated_usd"] == pytest.approx(family["missing"] * (1000 * 0.44 + 300 * 1.32) / 1_000_000)
+
+
 class FixedCostChat(ScriptedChat):
     def complete(self, **kwargs: Any) -> Any:
         response = super().complete(**kwargs)
@@ -181,6 +224,7 @@ def test_cli_dry_run_needs_no_endpoint_and_does_not_write_out(
     assert code == 0 and not out.exists() and snapshot(tmp_path) == before
     report = json.loads(capsys.readouterr().out)
     assert report["client_calls"] == 0 and report["status"] == "dry_run"
+    assert all(f["estimate_method"] == "estimated" for f in report["families"].values())
 
 
 def test_growing_call_costs_stay_inside_the_initial_reserve(tmp_path: Path) -> None:
@@ -200,20 +244,19 @@ def test_growing_call_costs_stay_inside_the_initial_reserve(tmp_path: Path) -> N
     assert total(report, "filled") == len(chat.calls) and total(report, "still_missing") > 0
 
 
-def test_dry_run_prices_prompt_tokens_and_family_cached_output_mean(tmp_path: Path) -> None:
+def test_dry_run_prices_prompt_tokens_with_empty_cache(tmp_path: Path) -> None:
     from openultrasast.learn.fill import DEFAULT_OUTPUT_TOKENS
 
     store, manifest, units = prepared(tmp_path)
-    seed_one(store, manifest, units[0])
     report = fill_rule(store, manifest, EXAMPLES, PRICES, excerpt_text, dry_run=True)
     folds = {f.name: f for f in ex.program_folds(EXAMPLES, manifest)[0]}
     index = {e.id: e for e in EXAMPLES}
     expected = dict.fromkeys(manifest.families, 0.0)
-    for unit in units[1:]:
+    for unit in units:
         example = index[unit.unit]
         program = Program(ex.arm_spec(manifest, manifest.arms["A"], unit.family, store), EXAMPLES, excerpt_text)
         prompt, _ = program.prepare(Candidate(Target.of(example), CODE[example.excerpt_sha], "python"), folds[unit.fold], seed=0)
-        output_tokens = 60 if unit.family == units[0].family else DEFAULT_OUTPUT_TOKENS
+        output_tokens = DEFAULT_OUTPUT_TOKENS
         expected[unit.family] += (
             program.spec.k * (prompt.estimated_tokens * PRICES["input_per_m"] + output_tokens * PRICES["output_per_m"]) / 1_000_000
         )