mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5.5
b31c85678f
Zero-cost replay of the six slice-2 programs: meter 0 client calls, $0.0; 211 of 353 units scored, 142 excluded for an uncached response (path lost all 39). Arm A (calibrated) fails calibration (ECE 0.077 access control, 0.1067 injection); arm B (Wilson LB >= 0.95, nested) chooses no threshold on any fold, as the registration's power note predicted (73 clean flags needed, 38 paired injection positives); secondary arm C flags nothing either. The instrument check shows real scores; the zero is the rule's. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
982 lines
26 KiB
JSON
982 lines
26 KiB
JSON
{
|
|
"arms": {
|
|
"A": {
|
|
"label": "today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p",
|
|
"rule": {
|
|
"name": "calibrated_block"
|
|
}
|
|
},
|
|
"B": {
|
|
"label": "selective blocking -- per-family raw-score threshold from the other folds' paired units, lowest t with Wilson 95% LB of flagged precision >= 0.95",
|
|
"rule": {
|
|
"bound": "wilson",
|
|
"name": "precision_bound_block",
|
|
"precision": 0.95,
|
|
"select_on": "paired"
|
|
}
|
|
},
|
|
"C": {
|
|
"label": "secondary, non-deciding -- as B with point precision >= 0.95 over at least 10 training flags",
|
|
"rule": {
|
|
"bound": "point",
|
|
"min_flagged": 10,
|
|
"name": "precision_bound_block",
|
|
"precision": 0.95,
|
|
"select_on": "paired"
|
|
}
|
|
}
|
|
},
|
|
"code_commit": "7bb97ad0bca62b5f986c5768297d8b8fca50fae4",
|
|
"decision": "inconclusive",
|
|
"experiment": "exp-003-precision-bound-blocking",
|
|
"families": {
|
|
"access_control": {
|
|
"A": {
|
|
"calibration_holds": false,
|
|
"ece": 0.077,
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 16,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 16,
|
|
"true_flagged": 0,
|
|
"units": 32
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 24,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 25,
|
|
"true_flagged": 0,
|
|
"units": 41
|
|
},
|
|
"reason_by_fold": {
|
|
"outer-0": "calibration",
|
|
"outer-1": "calibration",
|
|
"outer-2": "calibration",
|
|
"outer-3": "calibration",
|
|
"outer-4": "calibration"
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"B": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 16,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 16,
|
|
"true_flagged": 0,
|
|
"units": 32
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 24,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 25,
|
|
"true_flagged": 0,
|
|
"units": 41
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"C": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 16,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 16,
|
|
"true_flagged": 0,
|
|
"units": 32
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 24,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 25,
|
|
"true_flagged": 0,
|
|
"units": 41
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"capacity": {
|
|
"evaluated": 41,
|
|
"excluded_replay_miss": 0,
|
|
"not_run": 0,
|
|
"paired_positives_by_training_folds": {
|
|
"outer-0": 12,
|
|
"outer-1": 13,
|
|
"outer-2": 13,
|
|
"outer-3": 13,
|
|
"outer-4": 13
|
|
}
|
|
}
|
|
},
|
|
"deserialization": {
|
|
"A": {
|
|
"calibration_holds": false,
|
|
"ece": null,
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 3,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 2,
|
|
"true_flagged": 0,
|
|
"units": 5
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 12,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 11,
|
|
"true_flagged": 0,
|
|
"units": 14
|
|
},
|
|
"reason_by_fold": {
|
|
"outer-0": "calibration",
|
|
"outer-2": "calibration",
|
|
"outer-3": "calibration",
|
|
"outer-4": "calibration"
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"B": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 3,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 2,
|
|
"true_flagged": 0,
|
|
"units": 5
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 12,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 11,
|
|
"true_flagged": 0,
|
|
"units": 14
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"C": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 3,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 2,
|
|
"true_flagged": 0,
|
|
"units": 5
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 12,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 11,
|
|
"true_flagged": 0,
|
|
"units": 14
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"capacity": {
|
|
"evaluated": 14,
|
|
"excluded_replay_miss": 34,
|
|
"not_run": 0,
|
|
"paired_positives_by_training_folds": {
|
|
"outer-0": 0,
|
|
"outer-2": 2,
|
|
"outer-3": 2,
|
|
"outer-4": 2
|
|
}
|
|
}
|
|
},
|
|
"injection": {
|
|
"A": {
|
|
"calibration_holds": false,
|
|
"ece": 0.1067,
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 26,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 38,
|
|
"true_flagged": 0,
|
|
"units": 76
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 34,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 63,
|
|
"true_flagged": 0,
|
|
"units": 119
|
|
},
|
|
"reason_by_fold": {
|
|
"outer-0": "calibration",
|
|
"outer-1": "calibration",
|
|
"outer-2": "calibration",
|
|
"outer-3": "calibration",
|
|
"outer-4": "calibration"
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"B": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 26,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 38,
|
|
"true_flagged": 0,
|
|
"units": 76
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 34,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 63,
|
|
"true_flagged": 0,
|
|
"units": 119
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"C": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 26,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 38,
|
|
"true_flagged": 0,
|
|
"units": 76
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 34,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 63,
|
|
"true_flagged": 0,
|
|
"units": 119
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"capacity": {
|
|
"evaluated": 119,
|
|
"excluded_replay_miss": 0,
|
|
"not_run": 0,
|
|
"paired_positives_by_training_folds": {
|
|
"outer-0": 29,
|
|
"outer-1": 33,
|
|
"outer-2": 33,
|
|
"outer-3": 26,
|
|
"outer-4": 31
|
|
}
|
|
}
|
|
},
|
|
"output_encoding": {
|
|
"A": {
|
|
"calibration_holds": false,
|
|
"ece": null,
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 8,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 8,
|
|
"true_flagged": 0,
|
|
"units": 16
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 15,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 16,
|
|
"true_flagged": 0,
|
|
"units": 24
|
|
},
|
|
"reason_by_fold": {
|
|
"outer-0": "calibration",
|
|
"outer-1": "calibration",
|
|
"outer-2": "calibration",
|
|
"outer-3": "calibration",
|
|
"outer-4": "calibration"
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"B": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 8,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 8,
|
|
"true_flagged": 0,
|
|
"units": 16
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 15,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 16,
|
|
"true_flagged": 0,
|
|
"units": 24
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"C": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 8,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 8,
|
|
"true_flagged": 0,
|
|
"units": 16
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 15,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 16,
|
|
"true_flagged": 0,
|
|
"units": 24
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null,
|
|
"outer-4": null
|
|
}
|
|
},
|
|
"capacity": {
|
|
"evaluated": 24,
|
|
"excluded_replay_miss": 21,
|
|
"not_run": 0,
|
|
"paired_positives_by_training_folds": {
|
|
"outer-0": 7,
|
|
"outer-1": 6,
|
|
"outer-2": 6,
|
|
"outer-3": 7,
|
|
"outer-4": 6
|
|
}
|
|
}
|
|
},
|
|
"path": {
|
|
"A": {
|
|
"paired": {
|
|
"coverage": null,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 0,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 0,
|
|
"true_flagged": 0,
|
|
"units": 0
|
|
},
|
|
"pooled": {
|
|
"coverage": null,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 0,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 0,
|
|
"true_flagged": 0,
|
|
"units": 0
|
|
}
|
|
},
|
|
"B": {
|
|
"paired": {
|
|
"coverage": null,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 0,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 0,
|
|
"true_flagged": 0,
|
|
"units": 0
|
|
},
|
|
"pooled": {
|
|
"coverage": null,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 0,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 0,
|
|
"true_flagged": 0,
|
|
"units": 0
|
|
}
|
|
},
|
|
"C": {
|
|
"paired": {
|
|
"coverage": null,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 0,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 0,
|
|
"true_flagged": 0,
|
|
"units": 0
|
|
},
|
|
"pooled": {
|
|
"coverage": null,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 0,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 0,
|
|
"true_flagged": 0,
|
|
"units": 0
|
|
}
|
|
},
|
|
"capacity": {
|
|
"evaluated": 0,
|
|
"excluded_replay_miss": 39,
|
|
"not_run": 0,
|
|
"paired_positives_by_training_folds": {}
|
|
}
|
|
},
|
|
"untrusted_destination": {
|
|
"A": {
|
|
"calibration_holds": false,
|
|
"ece": null,
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 6,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 4,
|
|
"true_flagged": 0,
|
|
"units": 9
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 9,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 7,
|
|
"true_flagged": 0,
|
|
"units": 13
|
|
},
|
|
"reason_by_fold": {
|
|
"outer-0": "calibration",
|
|
"outer-1": "calibration",
|
|
"outer-2": "calibration",
|
|
"outer-3": "calibration"
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null
|
|
}
|
|
},
|
|
"B": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 6,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 4,
|
|
"true_flagged": 0,
|
|
"units": 9
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 9,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 7,
|
|
"true_flagged": 0,
|
|
"units": 13
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null
|
|
}
|
|
},
|
|
"C": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 6,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 4,
|
|
"true_flagged": 0,
|
|
"units": 9
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 9,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 7,
|
|
"true_flagged": 0,
|
|
"units": 13
|
|
},
|
|
"thresholds_by_fold": {
|
|
"outer-0": null,
|
|
"outer-1": null,
|
|
"outer-2": null,
|
|
"outer-3": null
|
|
}
|
|
},
|
|
"capacity": {
|
|
"evaluated": 13,
|
|
"excluded_replay_miss": 48,
|
|
"not_run": 0,
|
|
"paired_positives_by_training_folds": {
|
|
"outer-0": 4,
|
|
"outer-1": 3,
|
|
"outer-2": 3,
|
|
"outer-3": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"hypothesis": "The calibrated-probability BLOCK rule (Platt map + calibration gate, ECE <= 0.05) is the wrong instrument for M4, which asks for precision on what is flagged. Selective blocking -- a per-family raw-score threshold chosen on training folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95 -- reaches the M4 precision bar on held-out repositories where the calibrated rule offers no BLOCK.\n",
|
|
"instrument_check": {
|
|
"families": {
|
|
"access_control": {
|
|
"distinct_s": 30,
|
|
"paired": {
|
|
"blockable": 28,
|
|
"positives_before_first_negative": 10
|
|
},
|
|
"pooled": {
|
|
"blockable": 37,
|
|
"positives_before_first_negative": 19
|
|
},
|
|
"verdicts": {
|
|
"not_vulnerable": 19,
|
|
"unsure": 4,
|
|
"vulnerable": 18
|
|
}
|
|
},
|
|
"deserialization": {
|
|
"distinct_s": 5,
|
|
"paired": {
|
|
"blockable": 4,
|
|
"positives_before_first_negative": 1
|
|
},
|
|
"pooled": {
|
|
"blockable": 13,
|
|
"positives_before_first_negative": 10
|
|
},
|
|
"verdicts": {
|
|
"not_vulnerable": 2,
|
|
"unsure": 1,
|
|
"vulnerable": 11
|
|
}
|
|
},
|
|
"injection": {
|
|
"distinct_s": 48,
|
|
"paired": {
|
|
"blockable": 50,
|
|
"positives_before_first_negative": 3
|
|
},
|
|
"pooled": {
|
|
"blockable": 77,
|
|
"positives_before_first_negative": 11
|
|
},
|
|
"verdicts": {
|
|
"not_vulnerable": 46,
|
|
"unsure": 42,
|
|
"vulnerable": 31
|
|
}
|
|
},
|
|
"output_encoding": {
|
|
"distinct_s": 14,
|
|
"paired": {
|
|
"blockable": 11,
|
|
"positives_before_first_negative": 0
|
|
},
|
|
"pooled": {
|
|
"blockable": 19,
|
|
"positives_before_first_negative": 8
|
|
},
|
|
"verdicts": {
|
|
"not_vulnerable": 9,
|
|
"unsure": 5,
|
|
"vulnerable": 10
|
|
}
|
|
},
|
|
"untrusted_destination": {
|
|
"distinct_s": 9,
|
|
"paired": {
|
|
"blockable": 4,
|
|
"positives_before_first_negative": 1
|
|
},
|
|
"pooled": {
|
|
"blockable": 8,
|
|
"positives_before_first_negative": 4
|
|
},
|
|
"verdicts": {
|
|
"not_vulnerable": 3,
|
|
"unsure": 5,
|
|
"vulnerable": 5
|
|
}
|
|
}
|
|
},
|
|
"note": "descriptive, all folds together, ranking by raw score s among non-unsure verdicts; not nested, chooses nothing. It shows the scores are real and the zero is the rule's, not an unread input."
|
|
},
|
|
"manifest_digest": "5917f9a12035f09eae76978f3b5827b9fe59c8a9a5275f9ab54c3c1b55e8db3b",
|
|
"meter": {
|
|
"client_calls": 0,
|
|
"decided": 211,
|
|
"excluded_replay_miss": 142,
|
|
"folds_digest": "a1f78e141a572a4adcc9cd57eb8a784c440c177f0b60d5ef5bed9e94c8753c5f",
|
|
"metered_usd": 0.0,
|
|
"replay_priced_usd_not_spent": 0.582876,
|
|
"replayed_responses": 1056
|
|
},
|
|
"min_flagged_for_bound": 73,
|
|
"overall": {
|
|
"A": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 59,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 68,
|
|
"true_flagged": 0,
|
|
"units": 138
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 89,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 122,
|
|
"true_flagged": 0,
|
|
"units": 211
|
|
}
|
|
},
|
|
"B": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 59,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 68,
|
|
"true_flagged": 0,
|
|
"units": 138
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 89,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 122,
|
|
"true_flagged": 0,
|
|
"units": 211
|
|
}
|
|
},
|
|
"C": {
|
|
"paired": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 59,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 68,
|
|
"true_flagged": 0,
|
|
"units": 138
|
|
},
|
|
"pooled": {
|
|
"coverage": 0.0,
|
|
"flagged": 0,
|
|
"flagged_precision": null,
|
|
"flagged_precision_ci": null,
|
|
"flagged_precision_lb": null,
|
|
"groups": 89,
|
|
"meets_target_lb": false,
|
|
"meets_target_point": false,
|
|
"positives": 122,
|
|
"true_flagged": 0,
|
|
"units": 211
|
|
}
|
|
}
|
|
},
|
|
"populations": {
|
|
"primary": "paired",
|
|
"secondary": [
|
|
"pooled"
|
|
]
|
|
},
|
|
"primary": [
|
|
{
|
|
"name": "flagged_precision_lb",
|
|
"target": 0.95
|
|
}
|
|
],
|
|
"reason": "B offers no BLOCK on held-out paired units in any family; the incumbent stays",
|
|
"resamples": 2000,
|
|
"secondary": [
|
|
{
|
|
"name": "flagged_precision"
|
|
},
|
|
{
|
|
"name": "coverage"
|
|
},
|
|
{
|
|
"name": "flagged"
|
|
}
|
|
],
|
|
"spend_usd": {
|
|
"score": 0.0
|
|
},
|
|
"target_precision": 0.95,
|
|
"units": {
|
|
"by_label": {
|
|
"0": 89,
|
|
"1": 122
|
|
},
|
|
"evaluated": 211,
|
|
"excluded_replay_miss": 142,
|
|
"frozen": 353,
|
|
"groups": 89,
|
|
"not_run": 0,
|
|
"paired_evaluated": 138,
|
|
"unsure": 57
|
|
}
|
|
}
|