marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

exp-003: result -- inconclusive, no BLOCK rule flags a held-out unit

Browse Source 
Zero-cost replay of the six slice-2 programs: meter 0 client calls, $0.0;
211 of 353 units scored, 142 excluded for an uncached response (path lost
all 39). Arm A (calibrated) fails calibration (ECE 0.077 access control,
0.1067 injection); arm B (Wilson LB >= 0.95, nested) chooses no threshold on
any fold, as the registration's power note predicted (73 clean flags needed,
38 paired injection positives); secondary arm C flags nothing either. The
instrument check shows real scores; the zero is the rule's.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-03 08:15:22 +02:00
co-authored by Claude Opus 5.5
parent 7bb97ad0bc
commit b31c85678f
3 changed files with 1029 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/experiments/exp-003-precision-bound-blocking/registration.json
+1 -1
View File
@@ -35,5 +35,5 @@
"decision_rule": "REJECT B if in any family B flags held-out paired units at a point precision < 0.95; else ADOPT B if in some family B's held-out paired Wilson LB >= 0.95; else INCONCLUSIVE (the incumbent rule stays)",
"m4_reading": "a family reaches the M4 precision bar on held-out repositories when its held-out paired Wilson LB >= 0.95; point precision reported beside it",
"power": "a Wilson LB of 0.95 needs >= 73 flagged units with no false flag; the largest family holds 38 paired positives over all five folds, so arm B cannot offer a BLOCK in any family by construction; expected verdict inconclusive",
"status": "registered; not run"
"status": "run and analysed 2026-10-03: inconclusive -- no arm flags any held-out unit (result.json); meter 0 client calls, $0.0; 142 of 353 units excluded for an uncached response"
}
benchmarks/experiments/exp-003-precision-bound-blocking/result.json
+981
View File
@@ -0,0 +1,981 @@
{
"arms": {
"A": {
"label": "today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p",
"rule": {
"name": "calibrated_block"
}
},
"B": {
"label": "selective blocking -- per-family raw-score threshold from the other folds' paired units, lowest t with Wilson 95% LB of flagged precision >= 0.95",
"rule": {
"bound": "wilson",
"name": "precision_bound_block",
"precision": 0.95,
"select_on": "paired"
}
},
"C": {
"label": "secondary, non-deciding -- as B with point precision >= 0.95 over at least 10 training flags",
"rule": {
"bound": "point",
"min_flagged": 10,
"name": "precision_bound_block",
"precision": 0.95,
"select_on": "paired"
}
}
},
"code_commit": "7bb97ad0bca62b5f986c5768297d8b8fca50fae4",
"decision": "inconclusive",
"experiment": "exp-003-precision-bound-blocking",
"families": {
"access_control": {
"A": {
"calibration_holds": false,
"ece": 0.077,
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 16,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 16,
"true_flagged": 0,
"units": 32
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 24,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 25,
"true_flagged": 0,
"units": 41
},
"reason_by_fold": {
"outer-0": "calibration",
"outer-1": "calibration",
"outer-2": "calibration",
"outer-3": "calibration",
"outer-4": "calibration"
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"B": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 16,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 16,
"true_flagged": 0,
"units": 32
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 24,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 25,
"true_flagged": 0,
"units": 41
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"C": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 16,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 16,
"true_flagged": 0,
"units": 32
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 24,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 25,
"true_flagged": 0,
"units": 41
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"capacity": {
"evaluated": 41,
"excluded_replay_miss": 0,
"not_run": 0,
"paired_positives_by_training_folds": {
"outer-0": 12,
"outer-1": 13,
"outer-2": 13,
"outer-3": 13,
"outer-4": 13
}
}
},
"deserialization": {
"A": {
"calibration_holds": false,
"ece": null,
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 3,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 2,
"true_flagged": 0,
"units": 5
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 12,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 11,
"true_flagged": 0,
"units": 14
},
"reason_by_fold": {
"outer-0": "calibration",
"outer-2": "calibration",
"outer-3": "calibration",
"outer-4": "calibration"
},
"thresholds_by_fold": {
"outer-0": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"B": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 3,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 2,
"true_flagged": 0,
"units": 5
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 12,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 11,
"true_flagged": 0,
"units": 14
},
"thresholds_by_fold": {
"outer-0": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"C": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 3,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 2,
"true_flagged": 0,
"units": 5
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 12,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 11,
"true_flagged": 0,
"units": 14
},
"thresholds_by_fold": {
"outer-0": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"capacity": {
"evaluated": 14,
"excluded_replay_miss": 34,
"not_run": 0,
"paired_positives_by_training_folds": {
"outer-0": 0,
"outer-2": 2,
"outer-3": 2,
"outer-4": 2
}
}
},
"injection": {
"A": {
"calibration_holds": false,
"ece": 0.1067,
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 26,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 38,
"true_flagged": 0,
"units": 76
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 34,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 63,
"true_flagged": 0,
"units": 119
},
"reason_by_fold": {
"outer-0": "calibration",
"outer-1": "calibration",
"outer-2": "calibration",
"outer-3": "calibration",
"outer-4": "calibration"
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"B": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 26,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 38,
"true_flagged": 0,
"units": 76
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 34,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 63,
"true_flagged": 0,
"units": 119
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"C": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 26,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 38,
"true_flagged": 0,
"units": 76
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 34,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 63,
"true_flagged": 0,
"units": 119
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"capacity": {
"evaluated": 119,
"excluded_replay_miss": 0,
"not_run": 0,
"paired_positives_by_training_folds": {
"outer-0": 29,
"outer-1": 33,
"outer-2": 33,
"outer-3": 26,
"outer-4": 31
}
}
},
"output_encoding": {
"A": {
"calibration_holds": false,
"ece": null,
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 8,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 8,
"true_flagged": 0,
"units": 16
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 15,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 16,
"true_flagged": 0,
"units": 24
},
"reason_by_fold": {
"outer-0": "calibration",
"outer-1": "calibration",
"outer-2": "calibration",
"outer-3": "calibration",
"outer-4": "calibration"
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"B": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 8,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 8,
"true_flagged": 0,
"units": 16
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 15,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 16,
"true_flagged": 0,
"units": 24
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"C": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 8,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 8,
"true_flagged": 0,
"units": 16
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 15,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 16,
"true_flagged": 0,
"units": 24
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null,
"outer-4": null
}
},
"capacity": {
"evaluated": 24,
"excluded_replay_miss": 21,
"not_run": 0,
"paired_positives_by_training_folds": {
"outer-0": 7,
"outer-1": 6,
"outer-2": 6,
"outer-3": 7,
"outer-4": 6
}
}
},
"path": {
"A": {
"paired": {
"coverage": null,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 0,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 0,
"true_flagged": 0,
"units": 0
},
"pooled": {
"coverage": null,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 0,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 0,
"true_flagged": 0,
"units": 0
}
},
"B": {
"paired": {
"coverage": null,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 0,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 0,
"true_flagged": 0,
"units": 0
},
"pooled": {
"coverage": null,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 0,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 0,
"true_flagged": 0,
"units": 0
}
},
"C": {
"paired": {
"coverage": null,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 0,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 0,
"true_flagged": 0,
"units": 0
},
"pooled": {
"coverage": null,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 0,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 0,
"true_flagged": 0,
"units": 0
}
},
"capacity": {
"evaluated": 0,
"excluded_replay_miss": 39,
"not_run": 0,
"paired_positives_by_training_folds": {}
}
},
"untrusted_destination": {
"A": {
"calibration_holds": false,
"ece": null,
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 6,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 4,
"true_flagged": 0,
"units": 9
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 9,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 7,
"true_flagged": 0,
"units": 13
},
"reason_by_fold": {
"outer-0": "calibration",
"outer-1": "calibration",
"outer-2": "calibration",
"outer-3": "calibration"
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null
}
},
"B": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 6,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 4,
"true_flagged": 0,
"units": 9
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 9,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 7,
"true_flagged": 0,
"units": 13
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null
}
},
"C": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 6,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 4,
"true_flagged": 0,
"units": 9
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 9,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 7,
"true_flagged": 0,
"units": 13
},
"thresholds_by_fold": {
"outer-0": null,
"outer-1": null,
"outer-2": null,
"outer-3": null
}
},
"capacity": {
"evaluated": 13,
"excluded_replay_miss": 48,
"not_run": 0,
"paired_positives_by_training_folds": {
"outer-0": 4,
"outer-1": 3,
"outer-2": 3,
"outer-3": 2
}
}
}
},
"hypothesis": "The calibrated-probability BLOCK rule (Platt map + calibration gate, ECE <= 0.05) is the wrong instrument for M4, which asks for precision on what is flagged. Selective blocking -- a per-family raw-score threshold chosen on training folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95 -- reaches the M4 precision bar on held-out repositories where the calibrated rule offers no BLOCK.\n",
"instrument_check": {
"families": {
"access_control": {
"distinct_s": 30,
"paired": {
"blockable": 28,
"positives_before_first_negative": 10
},
"pooled": {
"blockable": 37,
"positives_before_first_negative": 19
},
"verdicts": {
"not_vulnerable": 19,
"unsure": 4,
"vulnerable": 18
}
},
"deserialization": {
"distinct_s": 5,
"paired": {
"blockable": 4,
"positives_before_first_negative": 1
},
"pooled": {
"blockable": 13,
"positives_before_first_negative": 10
},
"verdicts": {
"not_vulnerable": 2,
"unsure": 1,
"vulnerable": 11
}
},
"injection": {
"distinct_s": 48,
"paired": {
"blockable": 50,
"positives_before_first_negative": 3
},
"pooled": {
"blockable": 77,
"positives_before_first_negative": 11
},
"verdicts": {
"not_vulnerable": 46,
"unsure": 42,
"vulnerable": 31
}
},
"output_encoding": {
"distinct_s": 14,
"paired": {
"blockable": 11,
"positives_before_first_negative": 0
},
"pooled": {
"blockable": 19,
"positives_before_first_negative": 8
},
"verdicts": {
"not_vulnerable": 9,
"unsure": 5,
"vulnerable": 10
}
},
"untrusted_destination": {
"distinct_s": 9,
"paired": {
"blockable": 4,
"positives_before_first_negative": 1
},
"pooled": {
"blockable": 8,
"positives_before_first_negative": 4
},
"verdicts": {
"not_vulnerable": 3,
"unsure": 5,
"vulnerable": 5
}
}
},
"note": "descriptive, all folds together, ranking by raw score s among non-unsure verdicts; not nested, chooses nothing. It shows the scores are real and the zero is the rule's, not an unread input."
},
"manifest_digest": "5917f9a12035f09eae76978f3b5827b9fe59c8a9a5275f9ab54c3c1b55e8db3b",
"meter": {
"client_calls": 0,
"decided": 211,
"excluded_replay_miss": 142,
"folds_digest": "a1f78e141a572a4adcc9cd57eb8a784c440c177f0b60d5ef5bed9e94c8753c5f",
"metered_usd": 0.0,
"replay_priced_usd_not_spent": 0.582876,
"replayed_responses": 1056
},
"min_flagged_for_bound": 73,
"overall": {
"A": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 59,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 68,
"true_flagged": 0,
"units": 138
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 89,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 122,
"true_flagged": 0,
"units": 211
}
},
"B": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 59,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 68,
"true_flagged": 0,
"units": 138
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 89,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 122,
"true_flagged": 0,
"units": 211
}
},
"C": {
"paired": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 59,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 68,
"true_flagged": 0,
"units": 138
},
"pooled": {
"coverage": 0.0,
"flagged": 0,
"flagged_precision": null,
"flagged_precision_ci": null,
"flagged_precision_lb": null,
"groups": 89,
"meets_target_lb": false,
"meets_target_point": false,
"positives": 122,
"true_flagged": 0,
"units": 211
}
}
},
"populations": {
"primary": "paired",
"secondary": [
"pooled"
]
},
"primary": [
{
"name": "flagged_precision_lb",
"target": 0.95
}
],
"reason": "B offers no BLOCK on held-out paired units in any family; the incumbent stays",
"resamples": 2000,
"secondary": [
{
"name": "flagged_precision"
},
{
"name": "coverage"
},
{
"name": "flagged"
}
],
"spend_usd": {
"score": 0.0
},
"target_precision": 0.95,
"units": {
"by_label": {
"0": 89,
"1": 122
},
"evaluated": 211,
"excluded_replay_miss": 142,
"frozen": 353,
"groups": 89,
"not_run": 0,
"paired_evaluated": 138,
"unsure": 57
}
}
docs/decision-engine.md
+47 -1
View File
@@ -141,7 +141,7 @@ Changes to prompts, models, pass counts, features or sampling are compared by co
experiments. Each experiment comes from a pre-registered manifest
(`plane/experiments/<id>.yaml`). This way nothing is adopted on one run's number.
Two experiments are registered:
Three experiments are registered:
- exp-001 (known callers) has not run.
- exp-002 compared a retrieval ensemble against today's temperature sampling. It ran on 160
@@ -149,6 +149,52 @@ Two experiments are registered:
against 0.741 and the cost was $0.0021 against $0.0051 per candidate. The record shows re-run agreement
0.875 against 0.906 and an interval spanning zero
(`benchmarks/experiments/exp-002-retrieval-ensemble/result.json`).
- exp-003 compared two BLOCK rules on the same scores. It was **inconclusive**. Neither rule flags anything.
### exp-003: precision-bound blocking (2026-10-03)
The question: is the calibration gate what keeps BLOCK out of reach?
Arm A is today's rule: a calibrated probability, offered only when calibration holds.
Arm B picks a raw-score threshold per family, on training folds only.
It takes the lowest threshold whose Wilson 95% lower bound on precision reaches 0.95.
The held-out repository group is then scored with that threshold.
Arm C relaxes the bound to point precision 0.95 over at least 10 flags.
Arm C was registered as secondary and does not decide.
- **Registered first.** The manifest and units were committed and registered before the run.
- **Primary metric:** arm B's held-out Wilson lower bound on flagged precision, per family.
- **Primary population:** paired units only. Positives-only groups are excluded; pooled figures are secondary.
- **Zero cost.** Scores came only from the response cache, behind a zero-ceiling meter.
The meter recorded 0 client calls and $0.0.
- **Exclusions.** 142 of 353 units had an uncached response and were excluded, not asked.
Path lost all 39 units. Injection and access control lost none.
- **Evaluated:** 211 units in 89 repository groups, 138 of them paired.
| Family | Evaluated (excluded) | Paired positives | Flagged A / B / C | Coverage A / B / C |
| --- | --- | --- | --- | --- |
| access control | 41 (0) | 16 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 |
| deserialization | 14 (34) | 2 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 |
| injection | 119 (0) | 38 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 |
| output encoding | 24 (21) | 8 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 |
| path | 0 (39) | 0 | 0 / 0 / 0 | none |
| untrusted destination | 13 (48) | 4 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 |
No arm chose a threshold on any fold, so flagged precision is undefined everywhere.
Arm A failed calibration where it was checked: ECE 0.077 (access control), 0.1067 (injection).
The other families had too few units for a calibration check.
**Why B flags nothing.** A Wilson lower bound of 0.95 needs 73 flags without one error.
Injection, the largest family, holds 38 paired positives across all five folds.
The registration predicted this zero before the run.
On this data the bound, not the calibration gate, makes BLOCK unreachable.
**The zero is the rule's, not the instrument's.** The scores are real: 48 distinct values for injection.
In access control, the 10 highest-scored paired units are all positive.
In injection, a negative appears at rank 4 of the paired ranking.
Pooled rankings look cleaner because positives-only units fill the top.
Injection answered `unsure` for 42 of 119 units, and `unsure` never blocks.
Record: `benchmarks/experiments/exp-003-precision-bound-blocking/result.json`.
The reading is on [Where we stand](where-we-stand.md#2-our-approach-to-detection-step-by-step).