diff --git a/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py b/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py new file mode 100644 index 0000000..a347f5b --- /dev/null +++ b/benchmarks/experiments/exp-004-pooled-block-gate/instrument_check.py @@ -0,0 +1,36 @@ +"""Instrument check for exp-004: are the scores real, and how clean is the pooled top of the ranking?""" +import json, collections +from pathlib import Path +from openultrasast.config import load_dotenv +load_dotenv(Path(".env")) +from openultrasast.plane.memory import open_store +from openultrasast.learn import experiments as ex +store = open_store(None) +rows = [r.row for r in store.rows(repo=ex.experiment_repo("exp-004-pooled-block-gate"), kind="arm_outcome")] +units = {json.loads(l)["unit"]: json.loads(l) for l in open("plane/experiments/exp-004-pooled-block-gate.units.jsonl")} +print("arm_outcome rows", len(rows), "units file", len(units)) +r0 = rows[0]; print("row keys", sorted(r0)[:30]) +print("decision example", json.dumps(r0.get("decision"))[:300]) +def score(r): + d = r.get("decision") or {} + return r.get("s"), r.get("verdict", "unsure" if r.get("s") is None else "") +fam = collections.defaultdict(list) +for r in rows: + u = units.get(r.get("unit")) + if not u: continue + p, v = score(r) + fam[u["family"]].append((p, v, u["label"], u.get("pair"))) +for f, xs in sorted(fam.items()): + ps = [x for x in xs if x[0] is not None] + distinct = len({round(x[0], 4) for x in ps}) + unsure = sum(1 for x in xs if x[1] == "unsure") + pairs = collections.defaultdict(dict) + for p, v, y, pr in xs: + if pr and p is not None: pairs[pr][y] = p + full = [d for d in pairs.values() if 0 in d and 1 in d] + wins = sum(1 for d in full if d[1] > d[0]); ties = sum(1 for d in full if d[1] == d[0]) + top = sorted([x for x in ps if x[3]], key=lambda x: -x[0]) + first_neg = next((i for i, x in enumerate(top) if x[2] == 0), len(top)) + print(f"{f}: n={len(xs)} distinct_scores={distinct} unsure={unsure} pairs={len(full)} within_pair_win={wins} tie={ties} pos_above_first_neg={first_neg}") +allp = sorted([x for xs in fam.values() for x in xs if x[0] is not None and x[3]], key=lambda x: -x[0]) +print("pooled raw top-30 labels:", "".join(str(x[2]) for x in allp[:30])) diff --git a/benchmarks/experiments/exp-004-pooled-block-gate/record.json b/benchmarks/experiments/exp-004-pooled-block-gate/record.json new file mode 100644 index 0000000..8fcd18b --- /dev/null +++ b/benchmarks/experiments/exp-004-pooled-block-gate/record.json @@ -0,0 +1,81 @@ +{ + "experiment": "exp-004-pooled-block-gate", + "date": "2026-10-03", + "fill": { + "client_calls": 2073, + "usd": 1.238223, + "ceiling_usd": 2.0, + "still_missing": 0, + "dry_run_estimate_usd": 5.582, + "note": "the dry-run estimator over-priced 4.5x (no prompt-cache pricing); measured spend is the meter's" + }, + "run": { + "units": 501, + "client_calls": 0, + "usd": 0.0, + "replayed": 2509, + "replay_miss": 0 + }, + "verdict": "inconclusive", + "reason": "no t_block on any outer fold, pooled or per family; B offers no BLOCK", + "instrument_check": { + "per_family": { + "access_control": { + "n": 66, + "distinct_scores": 41, + "unsure": 5, + "pairs": 28, + "within_pair_wins": 22, + "ties": 1, + "positives_above_first_negative": 12 + }, + "deserialization": { + "n": 64, + "distinct_scores": 38, + "unsure": 7, + "pairs": 25, + "within_pair_wins": 18, + "ties": 4, + "positives_above_first_negative": 5 + }, + "injection": { + "n": 149, + "distinct_scores": 52, + "unsure": 54, + "pairs": 53, + "within_pair_wins": 30, + "ties": 17, + "positives_above_first_negative": 6 + }, + "output_encoding": { + "n": 69, + "distinct_scores": 31, + "unsure": 32, + "pairs": 28, + "within_pair_wins": 18, + "ties": 6, + "positives_above_first_negative": 2 + }, + "path": { + "n": 66, + "distinct_scores": 43, + "unsure": 15, + "pairs": 29, + "within_pair_wins": 25, + "ties": 3, + "positives_above_first_negative": 13 + }, + "untrusted_destination": { + "n": 87, + "distinct_scores": 36, + "unsure": 33, + "pairs": 32, + "within_pair_wins": 18, + "ties": 4, + "positives_above_first_negative": 3 + } + }, + "pooled_raw_top30_labels": "111111111110111110111111110111", + "reading": "scores are real (31-52 distinct values per family); pooled raw top 30 holds 27 positives (0.90), first negative at rank 12; the Wilson bound needs >= 73 flags with no error, so precision at the top, not data volume alone, now binds" + } +} \ No newline at end of file diff --git a/benchmarks/experiments/exp-004-pooled-block-gate/result.json b/benchmarks/experiments/exp-004-pooled-block-gate/result.json new file mode 100644 index 0000000..4d96898 --- /dev/null +++ b/benchmarks/experiments/exp-004-pooled-block-gate/result.json @@ -0,0 +1,822 @@ +{ + "arms": { + "A": { + "label": "today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p", + "rule": { + "name": "calibrated_block" + } + }, + "B": { + "label": "pooled gate -- one t_block over all families on a per-family Platt score (training folds, inner cross-fit), t_quiet per family at 90% recall, family floor 10 clean held-out flags", + "rule": { + "floor": 10, + "name": "pooled_block_gate", + "normalise": "platt", + "precision": 0.95, + "recall": 0.9, + "select_on": "paired" + } + } + }, + "code_commit": "90a2642019df414cb67b5dcea6e4ec235f35650a", + "decision": "inconclusive", + "experiment": "exp-004-pooled-block-gate", + "families": { + "access_control": { + "A": { + "calibration_holds": false, + "ece": 0.0715, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 28, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 28, + "true_flagged": 0, + "units": 56 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 37, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 66 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 28, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 28, + "true_flagged": 0, + "units": 56 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 37, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 66 + }, + "quiet_recall": { + "paired": 0.8929, + "pooled": 0.9211 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.44900641988143913, + "outer-1": 0.3723384028044276, + "outer-2": 0.4202772526614184, + "outer-3": 0.4481849709797749, + "outer-4": 0.36346174463707154 + } + }, + "capacity": { + "evaluated": 66, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 22, + "outer-1": 22, + "outer-2": 22, + "outer-3": 23, + "outer-4": 23 + } + } + }, + "deserialization": { + "A": { + "calibration_holds": false, + "ece": 0.0725, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 25, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 50 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 39, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 39, + "true_flagged": 0, + "units": 64 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 25, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 50 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 39, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 39, + "true_flagged": 0, + "units": 64 + }, + "quiet_recall": { + "paired": 0.88, + "pooled": 0.9231 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.17507194490500488, + "outer-1": 0.22766488322270723, + "outer-2": 0.2163745870743959, + "outer-3": 0.3112201878893598, + "outer-4": 0.363243722721776 + } + }, + "capacity": { + "evaluated": 64, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 20, + "outer-1": 20, + "outer-2": 20, + "outer-3": 20, + "outer-4": 20 + } + } + }, + "injection": { + "A": { + "calibration_holds": false, + "ece": 0.1042, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 41, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 53, + "true_flagged": 0, + "units": 106 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 49, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 78, + "true_flagged": 0, + "units": 149 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 41, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 53, + "true_flagged": 0, + "units": 106 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 49, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 78, + "true_flagged": 0, + "units": 149 + }, + "quiet_recall": { + "paired": 0.8679, + "pooled": 0.8974 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.287983504097999, + "outer-1": 0.17524866355153343, + "outer-2": 0.12918182665453967, + "outer-3": 0.14521040470628938, + "outer-4": 0.23005714687697673 + } + }, + "capacity": { + "evaluated": 149, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 38, + "outer-1": 44, + "outer-2": 45, + "outer-3": 43, + "outer-4": 42 + } + } + }, + "output_encoding": { + "A": { + "calibration_holds": false, + "ece": 0.0736, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 28, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 28, + "true_flagged": 0, + "units": 56 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 39, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 41, + "true_flagged": 0, + "units": 69 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 28, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 28, + "true_flagged": 0, + "units": 56 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 39, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 41, + "true_flagged": 0, + "units": 69 + }, + "quiet_recall": { + "paired": 0.8571, + "pooled": 0.9024 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.3723166587726901, + "outer-1": 0.5329072757249385, + "outer-2": 0.3785066578507705, + "outer-3": 0.4389011844848602, + "outer-4": 0.43855966643440264 + } + }, + "capacity": { + "evaluated": 69, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 23, + "outer-1": 22, + "outer-2": 22, + "outer-3": 22, + "outer-4": 23 + } + } + }, + "path": { + "A": { + "calibration_holds": false, + "ece": 0.0621, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 29, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 29, + "true_flagged": 0, + "units": 58 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 37, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 37, + "true_flagged": 0, + "units": 66 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 29, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 29, + "true_flagged": 0, + "units": 58 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 37, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 37, + "true_flagged": 0, + "units": 66 + }, + "quiet_recall": { + "paired": 0.8621, + "pooled": 0.8919 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.4680507207548809, + "outer-1": 0.3955717127313657, + "outer-2": 0.43694703358760895, + "outer-3": 0.4450583743716199, + "outer-4": 0.46396657833309796 + } + }, + "capacity": { + "evaluated": 66, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 23, + "outer-1": 23, + "outer-2": 24, + "outer-3": 23, + "outer-4": 23 + } + } + }, + "untrusted_destination": { + "A": { + "calibration_holds": false, + "ece": 0.1143, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 31, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 32, + "true_flagged": 0, + "units": 64 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 49, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 50, + "true_flagged": 0, + "units": 87 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "floor": { + "errors": 0, + "flagged": 0, + "floor": 10, + "may_block": false + }, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 31, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 32, + "true_flagged": 0, + "units": 64 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 49, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 50, + "true_flagged": 0, + "units": 87 + }, + "quiet_recall": { + "paired": 1.0, + "pooled": 1.0 + }, + "t_block_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + }, + "t_quiet_by_fold": { + "outer-0": 0.13676746287025585, + "outer-1": 0.11255775703980551, + "outer-2": 0.13763698150768486, + "outer-3": 0.09787938647374639, + "outer-4": 0.13763698150768486 + } + }, + "capacity": { + "evaluated": 87, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 26, + "outer-1": 26, + "outer-2": 24, + "outer-3": 26, + "outer-4": 26 + } + } + } + }, + "hypothesis": "One BLOCK threshold pooled across families on a per-family Platt-normalised score, chosen on training folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95, reaches that bound on held-out repositories' paired units, where per-family thresholds (exp-003) could not for lack of flags; a per-family floor (>= 10 held-out flags, none wrong) keeps a family that the pooling masks from blocking.\n", + "manifest_digest": "87fceb6580c6de629d6a6089eb5888cdfc4df0f791576184f987c14c192d85f5", + "min_flagged_for_bound": 73, + "overall": { + "A": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 182, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 195, + "true_flagged": 0, + "units": 390 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 220, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 283, + "true_flagged": 0, + "units": 501 + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 182, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 195, + "true_flagged": 0, + "units": 390 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 220, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 283, + "true_flagged": 0, + "units": 501 + } + } + }, + "overall_after_floor": { + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 182, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 195, + "true_flagged": 0, + "units": 390 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 220, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 283, + "true_flagged": 0, + "units": 501 + } + } + }, + "populations": { + "primary": "paired", + "secondary": [ + "pooled" + ] + }, + "primary": [ + { + "name": "flagged_precision_lb", + "target": 0.95 + } + ], + "reason": "B offers no BLOCK on held-out paired units; the incumbent stays", + "resamples": 2000, + "secondary": [ + { + "name": "flagged_precision" + }, + { + "name": "coverage" + }, + { + "name": "flagged" + }, + { + "name": "quiet_recall" + } + ], + "spend_usd": { + "score": 0.0 + }, + "target_precision": 0.95, + "units": { + "by_label": { + "0": 218, + "1": 283 + }, + "evaluated": 501, + "excluded_replay_miss": 0, + "frozen": 501, + "groups": 220, + "not_run": 0, + "paired_evaluated": 390, + "unsure": 146 + } +}