mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
learn: exp-004 run -- inconclusive, no pooled t_block; fill \$1.24 of a \$2 ceiling, replay \$0; instrument check recorded
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
90a2642019
commit
b50f7afccb
@@ -0,0 +1,36 @@
|
||||
"""Instrument check for exp-004: are the scores real, and how clean is the pooled top of the ranking?"""
|
||||
import json, collections
|
||||
from pathlib import Path
|
||||
from openultrasast.config import load_dotenv
|
||||
load_dotenv(Path(".env"))
|
||||
from openultrasast.plane.memory import open_store
|
||||
from openultrasast.learn import experiments as ex
|
||||
store = open_store(None)
|
||||
rows = [r.row for r in store.rows(repo=ex.experiment_repo("exp-004-pooled-block-gate"), kind="arm_outcome")]
|
||||
units = {json.loads(l)["unit"]: json.loads(l) for l in open("plane/experiments/exp-004-pooled-block-gate.units.jsonl")}
|
||||
print("arm_outcome rows", len(rows), "units file", len(units))
|
||||
r0 = rows[0]; print("row keys", sorted(r0)[:30])
|
||||
print("decision example", json.dumps(r0.get("decision"))[:300])
|
||||
def score(r):
|
||||
d = r.get("decision") or {}
|
||||
return r.get("s"), r.get("verdict", "unsure" if r.get("s") is None else "")
|
||||
fam = collections.defaultdict(list)
|
||||
for r in rows:
|
||||
u = units.get(r.get("unit"))
|
||||
if not u: continue
|
||||
p, v = score(r)
|
||||
fam[u["family"]].append((p, v, u["label"], u.get("pair")))
|
||||
for f, xs in sorted(fam.items()):
|
||||
ps = [x for x in xs if x[0] is not None]
|
||||
distinct = len({round(x[0], 4) for x in ps})
|
||||
unsure = sum(1 for x in xs if x[1] == "unsure")
|
||||
pairs = collections.defaultdict(dict)
|
||||
for p, v, y, pr in xs:
|
||||
if pr and p is not None: pairs[pr][y] = p
|
||||
full = [d for d in pairs.values() if 0 in d and 1 in d]
|
||||
wins = sum(1 for d in full if d[1] > d[0]); ties = sum(1 for d in full if d[1] == d[0])
|
||||
top = sorted([x for x in ps if x[3]], key=lambda x: -x[0])
|
||||
first_neg = next((i for i, x in enumerate(top) if x[2] == 0), len(top))
|
||||
print(f"{f}: n={len(xs)} distinct_scores={distinct} unsure={unsure} pairs={len(full)} within_pair_win={wins} tie={ties} pos_above_first_neg={first_neg}")
|
||||
allp = sorted([x for xs in fam.values() for x in xs if x[0] is not None and x[3]], key=lambda x: -x[0])
|
||||
print("pooled raw top-30 labels:", "".join(str(x[2]) for x in allp[:30]))
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"experiment": "exp-004-pooled-block-gate",
|
||||
"date": "2026-10-03",
|
||||
"fill": {
|
||||
"client_calls": 2073,
|
||||
"usd": 1.238223,
|
||||
"ceiling_usd": 2.0,
|
||||
"still_missing": 0,
|
||||
"dry_run_estimate_usd": 5.582,
|
||||
"note": "the dry-run estimator over-priced 4.5x (no prompt-cache pricing); measured spend is the meter's"
|
||||
},
|
||||
"run": {
|
||||
"units": 501,
|
||||
"client_calls": 0,
|
||||
"usd": 0.0,
|
||||
"replayed": 2509,
|
||||
"replay_miss": 0
|
||||
},
|
||||
"verdict": "inconclusive",
|
||||
"reason": "no t_block on any outer fold, pooled or per family; B offers no BLOCK",
|
||||
"instrument_check": {
|
||||
"per_family": {
|
||||
"access_control": {
|
||||
"n": 66,
|
||||
"distinct_scores": 41,
|
||||
"unsure": 5,
|
||||
"pairs": 28,
|
||||
"within_pair_wins": 22,
|
||||
"ties": 1,
|
||||
"positives_above_first_negative": 12
|
||||
},
|
||||
"deserialization": {
|
||||
"n": 64,
|
||||
"distinct_scores": 38,
|
||||
"unsure": 7,
|
||||
"pairs": 25,
|
||||
"within_pair_wins": 18,
|
||||
"ties": 4,
|
||||
"positives_above_first_negative": 5
|
||||
},
|
||||
"injection": {
|
||||
"n": 149,
|
||||
"distinct_scores": 52,
|
||||
"unsure": 54,
|
||||
"pairs": 53,
|
||||
"within_pair_wins": 30,
|
||||
"ties": 17,
|
||||
"positives_above_first_negative": 6
|
||||
},
|
||||
"output_encoding": {
|
||||
"n": 69,
|
||||
"distinct_scores": 31,
|
||||
"unsure": 32,
|
||||
"pairs": 28,
|
||||
"within_pair_wins": 18,
|
||||
"ties": 6,
|
||||
"positives_above_first_negative": 2
|
||||
},
|
||||
"path": {
|
||||
"n": 66,
|
||||
"distinct_scores": 43,
|
||||
"unsure": 15,
|
||||
"pairs": 29,
|
||||
"within_pair_wins": 25,
|
||||
"ties": 3,
|
||||
"positives_above_first_negative": 13
|
||||
},
|
||||
"untrusted_destination": {
|
||||
"n": 87,
|
||||
"distinct_scores": 36,
|
||||
"unsure": 33,
|
||||
"pairs": 32,
|
||||
"within_pair_wins": 18,
|
||||
"ties": 4,
|
||||
"positives_above_first_negative": 3
|
||||
}
|
||||
},
|
||||
"pooled_raw_top30_labels": "111111111110111110111111110111",
|
||||
"reading": "scores are real (31-52 distinct values per family); pooled raw top 30 holds 27 positives (0.90), first negative at rank 12; the Wilson bound needs >= 73 flags with no error, so precision at the top, not data volume alone, now binds"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,822 @@
|
||||
{
|
||||
"arms": {
|
||||
"A": {
|
||||
"label": "today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p",
|
||||
"rule": {
|
||||
"name": "calibrated_block"
|
||||
}
|
||||
},
|
||||
"B": {
|
||||
"label": "pooled gate -- one t_block over all families on a per-family Platt score (training folds, inner cross-fit), t_quiet per family at 90% recall, family floor 10 clean held-out flags",
|
||||
"rule": {
|
||||
"floor": 10,
|
||||
"name": "pooled_block_gate",
|
||||
"normalise": "platt",
|
||||
"precision": 0.95,
|
||||
"recall": 0.9,
|
||||
"select_on": "paired"
|
||||
}
|
||||
}
|
||||
},
|
||||
"code_commit": "90a2642019df414cb67b5dcea6e4ec235f35650a",
|
||||
"decision": "inconclusive",
|
||||
"experiment": "exp-004-pooled-block-gate",
|
||||
"families": {
|
||||
"access_control": {
|
||||
"A": {
|
||||
"calibration_holds": false,
|
||||
"ece": 0.0715,
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 28,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 28,
|
||||
"true_flagged": 0,
|
||||
"units": 56
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 37,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 38,
|
||||
"true_flagged": 0,
|
||||
"units": 66
|
||||
},
|
||||
"reason_by_fold": {
|
||||
"outer-0": "calibration",
|
||||
"outer-1": "calibration",
|
||||
"outer-2": "calibration",
|
||||
"outer-3": "calibration",
|
||||
"outer-4": "calibration"
|
||||
},
|
||||
"thresholds_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
}
|
||||
},
|
||||
"B": {
|
||||
"floor": {
|
||||
"errors": 0,
|
||||
"flagged": 0,
|
||||
"floor": 10,
|
||||
"may_block": false
|
||||
},
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 28,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 28,
|
||||
"true_flagged": 0,
|
||||
"units": 56
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 37,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 38,
|
||||
"true_flagged": 0,
|
||||
"units": 66
|
||||
},
|
||||
"quiet_recall": {
|
||||
"paired": 0.8929,
|
||||
"pooled": 0.9211
|
||||
},
|
||||
"t_block_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
},
|
||||
"t_quiet_by_fold": {
|
||||
"outer-0": 0.44900641988143913,
|
||||
"outer-1": 0.3723384028044276,
|
||||
"outer-2": 0.4202772526614184,
|
||||
"outer-3": 0.4481849709797749,
|
||||
"outer-4": 0.36346174463707154
|
||||
}
|
||||
},
|
||||
"capacity": {
|
||||
"evaluated": 66,
|
||||
"excluded_replay_miss": 0,
|
||||
"not_run": 0,
|
||||
"paired_positives_by_training_folds": {
|
||||
"outer-0": 22,
|
||||
"outer-1": 22,
|
||||
"outer-2": 22,
|
||||
"outer-3": 23,
|
||||
"outer-4": 23
|
||||
}
|
||||
}
|
||||
},
|
||||
"deserialization": {
|
||||
"A": {
|
||||
"calibration_holds": false,
|
||||
"ece": 0.0725,
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 25,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 25,
|
||||
"true_flagged": 0,
|
||||
"units": 50
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 39,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 39,
|
||||
"true_flagged": 0,
|
||||
"units": 64
|
||||
},
|
||||
"reason_by_fold": {
|
||||
"outer-0": "calibration",
|
||||
"outer-1": "calibration",
|
||||
"outer-2": "calibration",
|
||||
"outer-3": "calibration",
|
||||
"outer-4": "calibration"
|
||||
},
|
||||
"thresholds_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
}
|
||||
},
|
||||
"B": {
|
||||
"floor": {
|
||||
"errors": 0,
|
||||
"flagged": 0,
|
||||
"floor": 10,
|
||||
"may_block": false
|
||||
},
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 25,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 25,
|
||||
"true_flagged": 0,
|
||||
"units": 50
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 39,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 39,
|
||||
"true_flagged": 0,
|
||||
"units": 64
|
||||
},
|
||||
"quiet_recall": {
|
||||
"paired": 0.88,
|
||||
"pooled": 0.9231
|
||||
},
|
||||
"t_block_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
},
|
||||
"t_quiet_by_fold": {
|
||||
"outer-0": 0.17507194490500488,
|
||||
"outer-1": 0.22766488322270723,
|
||||
"outer-2": 0.2163745870743959,
|
||||
"outer-3": 0.3112201878893598,
|
||||
"outer-4": 0.363243722721776
|
||||
}
|
||||
},
|
||||
"capacity": {
|
||||
"evaluated": 64,
|
||||
"excluded_replay_miss": 0,
|
||||
"not_run": 0,
|
||||
"paired_positives_by_training_folds": {
|
||||
"outer-0": 20,
|
||||
"outer-1": 20,
|
||||
"outer-2": 20,
|
||||
"outer-3": 20,
|
||||
"outer-4": 20
|
||||
}
|
||||
}
|
||||
},
|
||||
"injection": {
|
||||
"A": {
|
||||
"calibration_holds": false,
|
||||
"ece": 0.1042,
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 41,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 53,
|
||||
"true_flagged": 0,
|
||||
"units": 106
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 49,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 78,
|
||||
"true_flagged": 0,
|
||||
"units": 149
|
||||
},
|
||||
"reason_by_fold": {
|
||||
"outer-0": "calibration",
|
||||
"outer-1": "calibration",
|
||||
"outer-2": "calibration",
|
||||
"outer-3": "calibration",
|
||||
"outer-4": "calibration"
|
||||
},
|
||||
"thresholds_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
}
|
||||
},
|
||||
"B": {
|
||||
"floor": {
|
||||
"errors": 0,
|
||||
"flagged": 0,
|
||||
"floor": 10,
|
||||
"may_block": false
|
||||
},
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 41,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 53,
|
||||
"true_flagged": 0,
|
||||
"units": 106
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 49,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 78,
|
||||
"true_flagged": 0,
|
||||
"units": 149
|
||||
},
|
||||
"quiet_recall": {
|
||||
"paired": 0.8679,
|
||||
"pooled": 0.8974
|
||||
},
|
||||
"t_block_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
},
|
||||
"t_quiet_by_fold": {
|
||||
"outer-0": 0.287983504097999,
|
||||
"outer-1": 0.17524866355153343,
|
||||
"outer-2": 0.12918182665453967,
|
||||
"outer-3": 0.14521040470628938,
|
||||
"outer-4": 0.23005714687697673
|
||||
}
|
||||
},
|
||||
"capacity": {
|
||||
"evaluated": 149,
|
||||
"excluded_replay_miss": 0,
|
||||
"not_run": 0,
|
||||
"paired_positives_by_training_folds": {
|
||||
"outer-0": 38,
|
||||
"outer-1": 44,
|
||||
"outer-2": 45,
|
||||
"outer-3": 43,
|
||||
"outer-4": 42
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_encoding": {
|
||||
"A": {
|
||||
"calibration_holds": false,
|
||||
"ece": 0.0736,
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 28,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 28,
|
||||
"true_flagged": 0,
|
||||
"units": 56
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 39,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 41,
|
||||
"true_flagged": 0,
|
||||
"units": 69
|
||||
},
|
||||
"reason_by_fold": {
|
||||
"outer-0": "calibration",
|
||||
"outer-1": "calibration",
|
||||
"outer-2": "calibration",
|
||||
"outer-3": "calibration",
|
||||
"outer-4": "calibration"
|
||||
},
|
||||
"thresholds_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
}
|
||||
},
|
||||
"B": {
|
||||
"floor": {
|
||||
"errors": 0,
|
||||
"flagged": 0,
|
||||
"floor": 10,
|
||||
"may_block": false
|
||||
},
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 28,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 28,
|
||||
"true_flagged": 0,
|
||||
"units": 56
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 39,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 41,
|
||||
"true_flagged": 0,
|
||||
"units": 69
|
||||
},
|
||||
"quiet_recall": {
|
||||
"paired": 0.8571,
|
||||
"pooled": 0.9024
|
||||
},
|
||||
"t_block_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
},
|
||||
"t_quiet_by_fold": {
|
||||
"outer-0": 0.3723166587726901,
|
||||
"outer-1": 0.5329072757249385,
|
||||
"outer-2": 0.3785066578507705,
|
||||
"outer-3": 0.4389011844848602,
|
||||
"outer-4": 0.43855966643440264
|
||||
}
|
||||
},
|
||||
"capacity": {
|
||||
"evaluated": 69,
|
||||
"excluded_replay_miss": 0,
|
||||
"not_run": 0,
|
||||
"paired_positives_by_training_folds": {
|
||||
"outer-0": 23,
|
||||
"outer-1": 22,
|
||||
"outer-2": 22,
|
||||
"outer-3": 22,
|
||||
"outer-4": 23
|
||||
}
|
||||
}
|
||||
},
|
||||
"path": {
|
||||
"A": {
|
||||
"calibration_holds": false,
|
||||
"ece": 0.0621,
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 29,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 29,
|
||||
"true_flagged": 0,
|
||||
"units": 58
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 37,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 37,
|
||||
"true_flagged": 0,
|
||||
"units": 66
|
||||
},
|
||||
"reason_by_fold": {
|
||||
"outer-0": "calibration",
|
||||
"outer-1": "calibration",
|
||||
"outer-2": "calibration",
|
||||
"outer-3": "calibration",
|
||||
"outer-4": "calibration"
|
||||
},
|
||||
"thresholds_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
}
|
||||
},
|
||||
"B": {
|
||||
"floor": {
|
||||
"errors": 0,
|
||||
"flagged": 0,
|
||||
"floor": 10,
|
||||
"may_block": false
|
||||
},
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 29,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 29,
|
||||
"true_flagged": 0,
|
||||
"units": 58
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 37,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 37,
|
||||
"true_flagged": 0,
|
||||
"units": 66
|
||||
},
|
||||
"quiet_recall": {
|
||||
"paired": 0.8621,
|
||||
"pooled": 0.8919
|
||||
},
|
||||
"t_block_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
},
|
||||
"t_quiet_by_fold": {
|
||||
"outer-0": 0.4680507207548809,
|
||||
"outer-1": 0.3955717127313657,
|
||||
"outer-2": 0.43694703358760895,
|
||||
"outer-3": 0.4450583743716199,
|
||||
"outer-4": 0.46396657833309796
|
||||
}
|
||||
},
|
||||
"capacity": {
|
||||
"evaluated": 66,
|
||||
"excluded_replay_miss": 0,
|
||||
"not_run": 0,
|
||||
"paired_positives_by_training_folds": {
|
||||
"outer-0": 23,
|
||||
"outer-1": 23,
|
||||
"outer-2": 24,
|
||||
"outer-3": 23,
|
||||
"outer-4": 23
|
||||
}
|
||||
}
|
||||
},
|
||||
"untrusted_destination": {
|
||||
"A": {
|
||||
"calibration_holds": false,
|
||||
"ece": 0.1143,
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 31,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 32,
|
||||
"true_flagged": 0,
|
||||
"units": 64
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 49,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 50,
|
||||
"true_flagged": 0,
|
||||
"units": 87
|
||||
},
|
||||
"reason_by_fold": {
|
||||
"outer-0": "calibration",
|
||||
"outer-1": "calibration",
|
||||
"outer-2": "calibration",
|
||||
"outer-3": "calibration",
|
||||
"outer-4": "calibration"
|
||||
},
|
||||
"thresholds_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
}
|
||||
},
|
||||
"B": {
|
||||
"floor": {
|
||||
"errors": 0,
|
||||
"flagged": 0,
|
||||
"floor": 10,
|
||||
"may_block": false
|
||||
},
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 31,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 32,
|
||||
"true_flagged": 0,
|
||||
"units": 64
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 49,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 50,
|
||||
"true_flagged": 0,
|
||||
"units": 87
|
||||
},
|
||||
"quiet_recall": {
|
||||
"paired": 1.0,
|
||||
"pooled": 1.0
|
||||
},
|
||||
"t_block_by_fold": {
|
||||
"outer-0": null,
|
||||
"outer-1": null,
|
||||
"outer-2": null,
|
||||
"outer-3": null,
|
||||
"outer-4": null
|
||||
},
|
||||
"t_quiet_by_fold": {
|
||||
"outer-0": 0.13676746287025585,
|
||||
"outer-1": 0.11255775703980551,
|
||||
"outer-2": 0.13763698150768486,
|
||||
"outer-3": 0.09787938647374639,
|
||||
"outer-4": 0.13763698150768486
|
||||
}
|
||||
},
|
||||
"capacity": {
|
||||
"evaluated": 87,
|
||||
"excluded_replay_miss": 0,
|
||||
"not_run": 0,
|
||||
"paired_positives_by_training_folds": {
|
||||
"outer-0": 26,
|
||||
"outer-1": 26,
|
||||
"outer-2": 24,
|
||||
"outer-3": 26,
|
||||
"outer-4": 26
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"hypothesis": "One BLOCK threshold pooled across families on a per-family Platt-normalised score, chosen on training folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95, reaches that bound on held-out repositories' paired units, where per-family thresholds (exp-003) could not for lack of flags; a per-family floor (>= 10 held-out flags, none wrong) keeps a family that the pooling masks from blocking.\n",
|
||||
"manifest_digest": "87fceb6580c6de629d6a6089eb5888cdfc4df0f791576184f987c14c192d85f5",
|
||||
"min_flagged_for_bound": 73,
|
||||
"overall": {
|
||||
"A": {
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 182,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 195,
|
||||
"true_flagged": 0,
|
||||
"units": 390
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 220,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 283,
|
||||
"true_flagged": 0,
|
||||
"units": 501
|
||||
}
|
||||
},
|
||||
"B": {
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 182,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 195,
|
||||
"true_flagged": 0,
|
||||
"units": 390
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 220,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 283,
|
||||
"true_flagged": 0,
|
||||
"units": 501
|
||||
}
|
||||
}
|
||||
},
|
||||
"overall_after_floor": {
|
||||
"B": {
|
||||
"paired": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 182,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 195,
|
||||
"true_flagged": 0,
|
||||
"units": 390
|
||||
},
|
||||
"pooled": {
|
||||
"coverage": 0.0,
|
||||
"flagged": 0,
|
||||
"flagged_precision": null,
|
||||
"flagged_precision_ci": null,
|
||||
"flagged_precision_lb": null,
|
||||
"groups": 220,
|
||||
"meets_target_lb": false,
|
||||
"meets_target_point": false,
|
||||
"positives": 283,
|
||||
"true_flagged": 0,
|
||||
"units": 501
|
||||
}
|
||||
}
|
||||
},
|
||||
"populations": {
|
||||
"primary": "paired",
|
||||
"secondary": [
|
||||
"pooled"
|
||||
]
|
||||
},
|
||||
"primary": [
|
||||
{
|
||||
"name": "flagged_precision_lb",
|
||||
"target": 0.95
|
||||
}
|
||||
],
|
||||
"reason": "B offers no BLOCK on held-out paired units; the incumbent stays",
|
||||
"resamples": 2000,
|
||||
"secondary": [
|
||||
{
|
||||
"name": "flagged_precision"
|
||||
},
|
||||
{
|
||||
"name": "coverage"
|
||||
},
|
||||
{
|
||||
"name": "flagged"
|
||||
},
|
||||
{
|
||||
"name": "quiet_recall"
|
||||
}
|
||||
],
|
||||
"spend_usd": {
|
||||
"score": 0.0
|
||||
},
|
||||
"target_precision": 0.95,
|
||||
"units": {
|
||||
"by_label": {
|
||||
"0": 218,
|
||||
"1": 283
|
||||
},
|
||||
"evaluated": 501,
|
||||
"excluded_replay_miss": 0,
|
||||
"frozen": 501,
|
||||
"groups": 220,
|
||||
"not_run": 0,
|
||||
"paired_evaluated": 390,
|
||||
"unsure": 146
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user