diff --git a/benchmarks/experiments/exp-003-precision-bound-blocking/registration.json b/benchmarks/experiments/exp-003-precision-bound-blocking/registration.json new file mode 100644 index 0000000..2707eb7 --- /dev/null +++ b/benchmarks/experiments/exp-003-precision-bound-blocking/registration.json @@ -0,0 +1,39 @@ +{ + "experiment": "exp-003-precision-bound-blocking", + "registered": "2026-10-03", + "store": "s3://sast-memory", + "manifest": "plane/experiments/exp-003-precision-bound-blocking.yaml", + "manifest_digest": "5917f9a12035f09eae76978f3b5827b9fe59c8a9a5275f9ab54c3c1b55e8db3b", + "pin": "dd7ebc119b5ad8e77c05d418eed9cbd46e193587", + "units_status": "frozen", + "units_file": "plane/experiments/exp-003-precision-bound-blocking.units.jsonl", + "units_digest": "338de998b1cfa36857cd793dbc4cf7fd0a13ab517a91f6790335881e941b3006", + "units": { + "total": 353, + "paired": 244, + "groups": 146, + "by_family": { + "access_control": {"units": 41, "positives": 25, "paired": 32, "paired_positives": 16, "groups": 24}, + "deserialization": {"units": 48, "positives": 31, "paired": 34, "paired_positives": 17, "groups": 31}, + "injection": {"units": 119, "positives": 63, "paired": 76, "paired_positives": 38, "groups": 34}, + "output_encoding": {"units": 45, "positives": 29, "paired": 32, "paired_positives": 16, "groups": 27}, + "path": {"units": 39, "positives": 23, "paired": 32, "paired_positives": 16, "groups": 23}, + "untrusted_destination": {"units": 61, "positives": 37, "paired": 38, "paired_positives": 19, "groups": 36} + }, + "note": "frozen model-free on the 937-example static memory; the outer folds (k 5, seed 0, compile fraction 0.25) are the incumbent evaluation's; injection and access_control equal exp-002's 160 units" + }, + "kind": "rule: one zero-cost scoring pass of the six slice-2 incumbent programs; arms differ only in the BLOCK rule", + "zero_cost": "responses replayed from the response cache only, behind a meter with zero USD and call ceilings around a refusing client; a unit with any uncached response is excluded and counted; the run is void unless the meter reads 0 calls and $0", + "arms": { + "A": "today's rule: Platt map cross-fitted per family; BLOCK only when the calibration check holds (slope CI contains 1, intercept CI contains 0, ECE <= 0.05); BLOCK point nested on the other folds' calibrated p (Wilson LB >= 0.95)", + "B": "selective blocking: per family and outer fold, threshold chosen on the family's paired units in the other four folds as the lowest raw score t with Wilson 95% LB of flagged precision >= 0.95 (unsure verdicts never flag); none -> no BLOCK on that fold", + "C": "secondary, non-deciding: as B with point precision >= 0.95 over at least 10 training flags" + }, + "primary_metric": "held-out (outer-fold) Wilson 95% lower bound of flagged precision, per family, arm B, paired population", + "populations": {"primary": "paired (both sides of the candidate present; excludes every positives-only group and any unpaired unit)", "secondary": ["pooled (every unit)"]}, + "secondary_metrics": ["flagged_precision", "coverage (flagged true positives / positives)", "flagged", "repository-cluster bootstrap 95% CI of flagged precision"], + "decision_rule": "REJECT B if in any family B flags held-out paired units at a point precision < 0.95; else ADOPT B if in some family B's held-out paired Wilson LB >= 0.95; else INCONCLUSIVE (the incumbent rule stays)", + "m4_reading": "a family reaches the M4 precision bar on held-out repositories when its held-out paired Wilson LB >= 0.95; point precision reported beside it", + "power": "a Wilson LB of 0.95 needs >= 73 flagged units with no false flag; the largest family holds 38 paired positives over all five folds, so arm B cannot offer a BLOCK in any family by construction; expected verdict inconclusive", + "status": "run and analysed 2026-10-03: inconclusive -- no arm flags any held-out unit (result.json); meter 0 client calls, $0.0; 142 of 353 units excluded for an uncached response" +} diff --git a/benchmarks/experiments/exp-003-precision-bound-blocking/result.json b/benchmarks/experiments/exp-003-precision-bound-blocking/result.json new file mode 100644 index 0000000..6a1ccd8 --- /dev/null +++ b/benchmarks/experiments/exp-003-precision-bound-blocking/result.json @@ -0,0 +1,981 @@ +{ + "arms": { + "A": { + "label": "today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p", + "rule": { + "name": "calibrated_block" + } + }, + "B": { + "label": "selective blocking -- per-family raw-score threshold from the other folds' paired units, lowest t with Wilson 95% LB of flagged precision >= 0.95", + "rule": { + "bound": "wilson", + "name": "precision_bound_block", + "precision": 0.95, + "select_on": "paired" + } + }, + "C": { + "label": "secondary, non-deciding -- as B with point precision >= 0.95 over at least 10 training flags", + "rule": { + "bound": "point", + "min_flagged": 10, + "name": "precision_bound_block", + "precision": 0.95, + "select_on": "paired" + } + } + }, + "code_commit": "7bb97ad0bca62b5f986c5768297d8b8fca50fae4", + "decision": "inconclusive", + "experiment": "exp-003-precision-bound-blocking", + "families": { + "access_control": { + "A": { + "calibration_holds": false, + "ece": 0.077, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 24, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 41 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 24, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 41 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 24, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 41 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 41, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 12, + "outer-1": 13, + "outer-2": 13, + "outer-3": 13, + "outer-4": 13 + } + } + }, + "deserialization": { + "A": { + "calibration_holds": false, + "ece": null, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 3, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 2, + "true_flagged": 0, + "units": 5 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 12, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 11, + "true_flagged": 0, + "units": 14 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 3, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 2, + "true_flagged": 0, + "units": 5 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 12, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 11, + "true_flagged": 0, + "units": 14 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 3, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 2, + "true_flagged": 0, + "units": 5 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 12, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 11, + "true_flagged": 0, + "units": 14 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 14, + "excluded_replay_miss": 34, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 0, + "outer-2": 2, + "outer-3": 2, + "outer-4": 2 + } + } + }, + "injection": { + "A": { + "calibration_holds": false, + "ece": 0.1067, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 26, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 76 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 34, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 63, + "true_flagged": 0, + "units": 119 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 26, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 76 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 34, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 63, + "true_flagged": 0, + "units": 119 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 26, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 76 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 34, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 63, + "true_flagged": 0, + "units": 119 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 119, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 29, + "outer-1": 33, + "outer-2": 33, + "outer-3": 26, + "outer-4": 31 + } + } + }, + "output_encoding": { + "A": { + "calibration_holds": false, + "ece": null, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 8, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 8, + "true_flagged": 0, + "units": 16 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 15, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 24 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 8, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 8, + "true_flagged": 0, + "units": 16 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 15, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 24 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 8, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 8, + "true_flagged": 0, + "units": 16 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 15, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 24 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 24, + "excluded_replay_miss": 21, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 7, + "outer-1": 6, + "outer-2": 6, + "outer-3": 7, + "outer-4": 6 + } + } + }, + "path": { + "A": { + "paired": { + "coverage": null, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 0, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 0, + "true_flagged": 0, + "units": 0 + }, + "pooled": { + "coverage": null, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 0, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 0, + "true_flagged": 0, + "units": 0 + } + }, + "B": { + "paired": { + "coverage": null, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 0, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 0, + "true_flagged": 0, + "units": 0 + }, + "pooled": { + "coverage": null, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 0, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 0, + "true_flagged": 0, + "units": 0 + } + }, + "C": { + "paired": { + "coverage": null, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 0, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 0, + "true_flagged": 0, + "units": 0 + }, + "pooled": { + "coverage": null, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 0, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 0, + "true_flagged": 0, + "units": 0 + } + }, + "capacity": { + "evaluated": 0, + "excluded_replay_miss": 39, + "not_run": 0, + "paired_positives_by_training_folds": {} + } + }, + "untrusted_destination": { + "A": { + "calibration_holds": false, + "ece": null, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 6, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 4, + "true_flagged": 0, + "units": 9 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 9, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 7, + "true_flagged": 0, + "units": 13 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 6, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 4, + "true_flagged": 0, + "units": 9 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 9, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 7, + "true_flagged": 0, + "units": 13 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 6, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 4, + "true_flagged": 0, + "units": 9 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 9, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 7, + "true_flagged": 0, + "units": 13 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null + } + }, + "capacity": { + "evaluated": 13, + "excluded_replay_miss": 48, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 4, + "outer-1": 3, + "outer-2": 3, + "outer-3": 2 + } + } + } + }, + "hypothesis": "The calibrated-probability BLOCK rule (Platt map + calibration gate, ECE <= 0.05) is the wrong instrument for M4, which asks for precision on what is flagged. Selective blocking -- a per-family raw-score threshold chosen on training folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95 -- reaches the M4 precision bar on held-out repositories where the calibrated rule offers no BLOCK.\n", + "instrument_check": { + "families": { + "access_control": { + "distinct_s": 30, + "paired": { + "blockable": 28, + "positives_before_first_negative": 10 + }, + "pooled": { + "blockable": 37, + "positives_before_first_negative": 19 + }, + "verdicts": { + "not_vulnerable": 19, + "unsure": 4, + "vulnerable": 18 + } + }, + "deserialization": { + "distinct_s": 5, + "paired": { + "blockable": 4, + "positives_before_first_negative": 1 + }, + "pooled": { + "blockable": 13, + "positives_before_first_negative": 10 + }, + "verdicts": { + "not_vulnerable": 2, + "unsure": 1, + "vulnerable": 11 + } + }, + "injection": { + "distinct_s": 48, + "paired": { + "blockable": 50, + "positives_before_first_negative": 3 + }, + "pooled": { + "blockable": 77, + "positives_before_first_negative": 11 + }, + "verdicts": { + "not_vulnerable": 46, + "unsure": 42, + "vulnerable": 31 + } + }, + "output_encoding": { + "distinct_s": 14, + "paired": { + "blockable": 11, + "positives_before_first_negative": 0 + }, + "pooled": { + "blockable": 19, + "positives_before_first_negative": 8 + }, + "verdicts": { + "not_vulnerable": 9, + "unsure": 5, + "vulnerable": 10 + } + }, + "untrusted_destination": { + "distinct_s": 9, + "paired": { + "blockable": 4, + "positives_before_first_negative": 1 + }, + "pooled": { + "blockable": 8, + "positives_before_first_negative": 4 + }, + "verdicts": { + "not_vulnerable": 3, + "unsure": 5, + "vulnerable": 5 + } + } + }, + "note": "descriptive, all folds together, ranking by raw score s among non-unsure verdicts; not nested, chooses nothing. It shows the scores are real and the zero is the rule's, not an unread input." + }, + "manifest_digest": "5917f9a12035f09eae76978f3b5827b9fe59c8a9a5275f9ab54c3c1b55e8db3b", + "meter": { + "client_calls": 0, + "decided": 211, + "excluded_replay_miss": 142, + "folds_digest": "a1f78e141a572a4adcc9cd57eb8a784c440c177f0b60d5ef5bed9e94c8753c5f", + "metered_usd": 0.0, + "replay_priced_usd_not_spent": 0.582876, + "replayed_responses": 1056 + }, + "min_flagged_for_bound": 73, + "overall": { + "A": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 59, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 68, + "true_flagged": 0, + "units": 138 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 89, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 122, + "true_flagged": 0, + "units": 211 + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 59, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 68, + "true_flagged": 0, + "units": 138 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 89, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 122, + "true_flagged": 0, + "units": 211 + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 59, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 68, + "true_flagged": 0, + "units": 138 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 89, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 122, + "true_flagged": 0, + "units": 211 + } + } + }, + "populations": { + "primary": "paired", + "secondary": [ + "pooled" + ] + }, + "primary": [ + { + "name": "flagged_precision_lb", + "target": 0.95 + } + ], + "reason": "B offers no BLOCK on held-out paired units in any family; the incumbent stays", + "resamples": 2000, + "secondary": [ + { + "name": "flagged_precision" + }, + { + "name": "coverage" + }, + { + "name": "flagged" + } + ], + "spend_usd": { + "score": 0.0 + }, + "target_precision": 0.95, + "units": { + "by_label": { + "0": 89, + "1": 122 + }, + "evaluated": 211, + "excluded_replay_miss": 142, + "frozen": 353, + "groups": 89, + "not_run": 0, + "paired_evaluated": 138, + "unsure": 57 + } +} diff --git a/docs/decision-engine.md b/docs/decision-engine.md index c63a696..d62e064 100644 --- a/docs/decision-engine.md +++ b/docs/decision-engine.md @@ -141,7 +141,7 @@ Changes to prompts, models, pass counts, features or sampling are compared by co experiments. Each experiment comes from a pre-registered manifest (`plane/experiments/.yaml`). This way nothing is adopted on one run's number. -Two experiments are registered: +Three experiments are registered: - exp-001 (known callers) has not run. - exp-002 compared a retrieval ensemble against today's temperature sampling. It ran on 160 @@ -149,6 +149,52 @@ Two experiments are registered: against 0.741 and the cost was $0.0021 against $0.0051 per candidate. The record shows re-run agreement 0.875 against 0.906 and an interval spanning zero (`benchmarks/experiments/exp-002-retrieval-ensemble/result.json`). +- exp-003 compared two BLOCK rules on the same scores. It was **inconclusive**. Neither rule flags anything. + +### exp-003: precision-bound blocking (2026-10-03) + +The question: is the calibration gate what keeps BLOCK out of reach? +Arm A is today's rule: a calibrated probability, offered only when calibration holds. +Arm B picks a raw-score threshold per family, on training folds only. +It takes the lowest threshold whose Wilson 95% lower bound on precision reaches 0.95. +The held-out repository group is then scored with that threshold. +Arm C relaxes the bound to point precision 0.95 over at least 10 flags. +Arm C was registered as secondary and does not decide. + +- **Registered first.** The manifest and units were committed and registered before the run. +- **Primary metric:** arm B's held-out Wilson lower bound on flagged precision, per family. +- **Primary population:** paired units only. Positives-only groups are excluded; pooled figures are secondary. +- **Zero cost.** Scores came only from the response cache, behind a zero-ceiling meter. + The meter recorded 0 client calls and $0.0. +- **Exclusions.** 142 of 353 units had an uncached response and were excluded, not asked. + Path lost all 39 units. Injection and access control lost none. +- **Evaluated:** 211 units in 89 repository groups, 138 of them paired. + +| Family | Evaluated (excluded) | Paired positives | Flagged A / B / C | Coverage A / B / C | +| --- | --- | --- | --- | --- | +| access control | 41 (0) | 16 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 | +| deserialization | 14 (34) | 2 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 | +| injection | 119 (0) | 38 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 | +| output encoding | 24 (21) | 8 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 | +| path | 0 (39) | 0 | 0 / 0 / 0 | none | +| untrusted destination | 13 (48) | 4 | 0 / 0 / 0 | 0.0 / 0.0 / 0.0 | + +No arm chose a threshold on any fold, so flagged precision is undefined everywhere. +Arm A failed calibration where it was checked: ECE 0.077 (access control), 0.1067 (injection). +The other families had too few units for a calibration check. + +**Why B flags nothing.** A Wilson lower bound of 0.95 needs 73 flags without one error. +Injection, the largest family, holds 38 paired positives across all five folds. +The registration predicted this zero before the run. +On this data the bound, not the calibration gate, makes BLOCK unreachable. + +**The zero is the rule's, not the instrument's.** The scores are real: 48 distinct values for injection. +In access control, the 10 highest-scored paired units are all positive. +In injection, a negative appears at rank 4 of the paired ranking. +Pooled rankings look cleaner because positives-only units fill the top. +Injection answered `unsure` for 42 of 119 units, and `unsure` never blocks. + +Record: `benchmarks/experiments/exp-003-precision-bound-blocking/result.json`. The reading is on [Where we stand](where-we-stand.md#2-our-approach-to-detection-step-by-step). diff --git a/plane/experiments/exp-003-precision-bound-blocking.units.jsonl b/plane/experiments/exp-003-precision-bound-blocking.units.jsonl new file mode 100644 index 0000000..4701d3a --- /dev/null +++ b/plane/experiments/exp-003-precision-bound-blocking.units.jsonl @@ -0,0 +1,353 @@ +{"family": "deserialization", "fold": "outer-1", "group": "aio-libs/aiohttp", "label": 1, "pair": "ee577ee4b6076fab", "unit": "2523f55d035d1a772a842090bc335e5d755b2fe15b376dd5b6b9e30b9dae82d1"} +{"family": "deserialization", "fold": "outer-1", "group": "aio-libs/aiohttp", "label": 0, "pair": "ee577ee4b6076fab", "unit": "ca0f6c160d7839204f6fcc1709bcbc749a02a0b3b8bb648e2a238eacbf22ed78"} +{"family": "access_control", "fold": "outer-0", "group": "alchemycms/alchemy_cms", "label": 1, "pair": "c95b6b01d0c2db2a", "unit": "15540c23be0c42472a49039658ec2ec0bbd8c6b318e35dd040a9d427370e0ef0"} +{"family": "access_control", "fold": "outer-0", "group": "alchemycms/alchemy_cms", "label": 0, "pair": "c95b6b01d0c2db2a", "unit": "542471980417e964182d7d3db59a05fe12bdd072449f3bd77f7daa5515dbc909"} +{"family": "access_control", "fold": "outer-1", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "bcc636b23dfb5f570d8864b17fe07b6b4fb52b65a40988c32011f1422f74d504"} +{"family": "access_control", "fold": "outer-1", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "dbcf1e67a1be7ccf7d51701f9b8339339164055135443044c23a446b16b3d934"} +{"family": "injection", "fold": "outer-1", "group": "anotherik/threatbyte", "label": 1, "pair": "", "unit": "50408f68114ecb4de3bebd836d52d9afad86b22dd1843641ac8a49b617c09e33"} +{"family": "injection", "fold": "outer-0", "group": "anxolerd/dvpwa", "label": 1, "pair": "", "unit": "cafb0f00f80dc6beeba40eb95c764bf9a3569193070ba321c28677dfa425ec88"} +{"family": "output_encoding", "fold": "outer-3", "group": "apache/logging-log4net", "label": 1, "pair": "fe0e104d111c2555", "unit": "088036fab9962be2b96f9162ea4f32190575678d3c5ba96bdfeac178bc22b2c4"} +{"family": "output_encoding", "fold": "outer-3", "group": "apache/logging-log4net", "label": 0, "pair": "fe0e104d111c2555", "unit": "7fd2b56a816e25358477eee64487fb2c77f34c652703ad0a7120c368492017a3"} +{"family": "path", "fold": "outer-0", "group": "apache/wicket", "label": 1, "pair": "b6742570f7f0e026", "unit": "7c6c1d27aee6557f1b6bf371b8209003f0d2d1122cfa35b6629eb43f38769cf4"} +{"family": "path", "fold": "outer-0", "group": "apache/wicket", "label": 0, "pair": "b6742570f7f0e026", "unit": "df8300d84dd784d0be46969ea432ef97b8ce611bae7a896ed4f349bd4d6897db"} +{"family": "access_control", "fold": "outer-0", "group": "apolloconfig/apollo", "label": 1, "pair": "0f584bb97a154488", "unit": "222a84ffc7c58d45e8f10df221762b23bf20d0287ecd20f375558134928886ca"} +{"family": "access_control", "fold": "outer-0", "group": "apolloconfig/apollo", "label": 0, "pair": "0f584bb97a154488", "unit": "25e4c9d2941f59f072385a7bde3fd0bd76f9e986ed24b06ce81962ab447f5094"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "archivy/archivy", "label": 0, "pair": "bb3d7baed62e6bb7", "unit": "e8ae88fd2b677e3f64ac94bddb012e3b903afa294ecc87774d10087c0d50489c"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "archivy/archivy", "label": 1, "pair": "bb3d7baed62e6bb7", "unit": "f6b0d241891dc1d897d74d98b6ff7ada2c1f1ce7874ca32097630c06ca53e389"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "axllent/mailpit", "label": 1, "pair": "143b900b99a46cf4", "unit": "200f96cd784ff5c39d8bba22883f22bc8a7c87c6970a7c871802f48a45548337"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "axllent/mailpit", "label": 0, "pair": "143b900b99a46cf4", "unit": "7a0d62ec424849ec4681f4333fbb077e8239054d1e964bd99b40847ad7ccdb01"} +{"family": "injection", "fold": "outer-2", "group": "browserify/static-eval", "label": 1, "pair": "c12dbc8a50f7fe9b", "unit": "04beb8a0fd054b92c2dd0ed83907330fdee6124d86114df948ac72a297a7d51f"} +{"family": "injection", "fold": "outer-2", "group": "browserify/static-eval", "label": 0, "pair": "c12dbc8a50f7fe9b", "unit": "c5029016a6fd2e3f9df59dba0ce3c173bbbb0d2b1de22d151086e01e1b272119"} +{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "0e213aeabacf4205db024ea3c557e720c3aba911424555784917e967d6404867"} +{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 0, "pair": "e8bc65415bd42e43", "unit": "4920230ef773595a9fd9a92a090dff42b48499a1dc5fc8831f941dae55cb2538"} +{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 1, "pair": "e8bc65415bd42e43", "unit": "49209a810f207eb61382e2ceeab7076936838a5652546f21c187d066e220160b"} +{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 0, "pair": "", "unit": "9e2964884cdf281fa3f18ee3481aa62f9a8a0a195d605da2df5bd71876fe311a"} +{"family": "injection", "fold": "outer-3", "group": "budibase/budibase", "label": 1, "pair": "", "unit": "f9cc6f1bbcd59f5dc0de8f96248f722ea0679f8afa47c53f70832151ce7616d3"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 0, "pair": "e69463f945f9a0f4", "unit": "5feace7b28e260175f80c682e9009025aea6fae3dc75409cabbb955b560d0372"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "cakephp/authentication", "label": 1, "pair": "e69463f945f9a0f4", "unit": "799a4f5a5b9324e86c90752f0239f059984f44e8ff1ec0dd518d0d9413e34c66"} +{"family": "deserialization", "fold": "outer-2", "group": "cesargb/laravel-magiclink", "label": 0, "pair": "768fc9db8a75a351", "unit": "adc6527469618f72b1dd7776814717cbd0056a0c625e05bef74199171ce73f1a"} +{"family": "deserialization", "fold": "outer-2", "group": "cesargb/laravel-magiclink", "label": 1, "pair": "768fc9db8a75a351", "unit": "da47663a57e370a75cc65b810b8a380aa26d12dcabb1d9b3c02641398ff7d348"} +{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 0, "pair": "fab4ce020049724b", "unit": "b9be90242be627aa46462143ac85940a906232ebc0b47776b5729475b27d215e"} +{"family": "path", "fold": "outer-0", "group": "christopy/serve-here.js", "label": 1, "pair": "fab4ce020049724b", "unit": "ece12ea7d87b4a12c4da30faed9eb530b51fc879c271b68ba5304b230cda8e79"} +{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 1, "pair": "9126ba83905eb8b4", "unit": "d8dc50b7bb40bbcfc3f3d4f1f8d3a4d17eb1c662d5f759efd18656a482dd2e32"} +{"family": "output_encoding", "fold": "outer-1", "group": "chrome-php/chrome", "label": 0, "pair": "9126ba83905eb8b4", "unit": "f426f9307621b3f2d6a8d1669cdb4c19e962b94e34fbb394d4660cbde5921c88"} +{"family": "deserialization", "fold": "outer-3", "group": "commenthol/serialize-to-js", "label": 0, "pair": "7ac5e21d9d8f4c47", "unit": "1d1efa2fd7323c264290d4d314c957c853e1c7a9c877a54d6f03ab7d2af2ca86"} +{"family": "deserialization", "fold": "outer-3", "group": "commenthol/serialize-to-js", "label": 1, "pair": "7ac5e21d9d8f4c47", "unit": "b5dc78c7ac0b76803dc98904142c1139083fb9daafb5bff40e7b5c3f14ee03f5"} +{"family": "deserialization", "fold": "outer-0", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "0894a7212280bab755cd9d93b1f0ede2c51e4fca76b76e5ed9e24ef8f8cf0b9c"} +{"family": "injection", "fold": "outer-0", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "02694b5fb85e9e52bf5eaaab4e0b628f1f44ff3a8d5ad026cc10afdbc4222eaf"} +{"family": "injection", "fold": "outer-0", "group": "contrast-security-oss/djangoat", "label": 1, "pair": "", "unit": "db9a3e888780501dd3c2c125af2f9d47f68ebf02f0e0db8370b2ea09c6f9d070"} +{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9af96b099dedcc4005e5fcd19ad78f73864e1cfbafbe6953b4edb6260957943d"} +{"family": "injection", "fold": "outer-1", "group": "contrast-security-oss/vulnpy", "label": 1, "pair": "", "unit": "9e7c2a786068a936a6c664cc229d6f39cea1fe57caf0806306f0cfc44515460c"} +{"family": "deserialization", "fold": "outer-4", "group": "cps-it/mailqueue", "label": 1, "pair": "07d1dc4d299a7700", "unit": "0e664e274dfb438d2f721edc81f8469ac48387dbd5c1b43973cd7e651db46db3"} +{"family": "deserialization", "fold": "outer-4", "group": "cps-it/mailqueue", "label": 0, "pair": "07d1dc4d299a7700", "unit": "da55f60378c67e91683b9720f2fa0e6659e28c7b392c050278b40713e47c84a3"} +{"family": "deserialization", "fold": "outer-0", "group": "datapizza-labs/datapizza-ai", "label": 1, "pair": "5a91049be21aa611", "unit": "1ab9936627aa59c3a60ea44074ad44b68aea4f9c04f14c1949dd9c77ca13ff0a"} +{"family": "deserialization", "fold": "outer-0", "group": "datapizza-labs/datapizza-ai", "label": 0, "pair": "5a91049be21aa611", "unit": "8df72b9eab3f2fd858c40312d0a806cb67c620bd62873826c8732ba8795d9cfa"} +{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 0, "pair": "", "unit": "115d87f37bf1d4ab91fa3010303e81b127f84b817268b60b8fd4b9934efeae4f"} +{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 0, "pair": "2b8101c89c839614", "unit": "7bb7d858a670fc51f81e08c141c2bca3b8b7524254ab679e33b22ce79536e895"} +{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 0, "pair": "c23acf4f38945093", "unit": "82bdc0b6bb0c19e342efa63af0c9014a607461d0682cdd14bd5fea33516fbd50"} +{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 1, "pair": "2b8101c89c839614", "unit": "886285661b88f14e00ff8e863b31cf7b7afb303ad4aa4aa13b1cffbf1272d5e0"} +{"family": "injection", "fold": "outer-0", "group": "davila7/claude-code-templates", "label": 1, "pair": "c23acf4f38945093", "unit": "bf74aae5a48156f183426a80cafca6da219ab13b2ae4b40fa88fca75acb72980"} +{"family": "access_control", "fold": "outer-2", "group": "delmaredigital/payload-puck", "label": 0, "pair": "d7cc37a3a54315cb", "unit": "4aecd43c6e5b81daf7aadd6a6e22f22c17c08b652a2f5121a49320ca1057b186"} +{"family": "access_control", "fold": "outer-2", "group": "delmaredigital/payload-puck", "label": 1, "pair": "d7cc37a3a54315cb", "unit": "88123ae860be2fdc28fd6a289cf26c3e0af0689f455cf8c15098e5dc76e5c8a8"} +{"family": "output_encoding", "fold": "outer-4", "group": "devbridge/jquery-autocomplete", "label": 0, "pair": "81caad96b221c2b8", "unit": "3c2dc720ed159e55629ba737665d93859def9790427289567a513c83c133e236"} +{"family": "output_encoding", "fold": "outer-4", "group": "devbridge/jquery-autocomplete", "label": 1, "pair": "81caad96b221c2b8", "unit": "e8b9595a320776c8e663fd0b168193c1e64e36a9052784d1219f2c4a91d647f2"} +{"family": "path", "fold": "outer-3", "group": "dgtlmoon/changedetection.io", "label": 0, "pair": "f9b6599018ef3b60", "unit": "7558d6ee39a311c5459e35fe6f7b12917b7f843b21ab1fbb5e485a07f53b7d78"} +{"family": "path", "fold": "outer-3", "group": "dgtlmoon/changedetection.io", "label": 1, "pair": "f9b6599018ef3b60", "unit": "b1a6cf450a074788ffca2bebd880435abac2c376e89cb328d7a22d277fdccad8"} +{"family": "output_encoding", "fold": "outer-0", "group": "dicebear/dicebear", "label": 1, "pair": "eaf2f03db2c502b5", "unit": "38dbfbf745d7cbcc52c6fff83248d84663273fe6a1f1b014eac7776c4bb8f074"} +{"family": "output_encoding", "fold": "outer-0", "group": "dicebear/dicebear", "label": 0, "pair": "eaf2f03db2c502b5", "unit": "a84d848ef56b72fc15872d88acaae1743c9cdf88095d13b1869abbec6a0d8a8f"} +{"family": "deserialization", "fold": "outer-1", "group": "docling-project/docling-core", "label": 0, "pair": "7df48bca2c9fabfc", "unit": "7b76ce8f55d61071adb367018cc3e50f04a05d08c7c4919c3806fc1cc8bf41a7"} +{"family": "deserialization", "fold": "outer-1", "group": "docling-project/docling-core", "label": 1, "pair": "7df48bca2c9fabfc", "unit": "c82ea0adc9c8366aa2b77ce73772038676fc37ff50879e873ff397958fdd7ec1"} +{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "7f02a09b8d8f2321f2cdade74c2fa1d52442e006149a749a8be768daaa62195d"} +{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "", "unit": "9b5e439943ce5c2b04d6201b7a843634aa1b13b726813c23450ff54ce03f3dd6"} +{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 1, "pair": "5b836889b4da4921", "unit": "a84c2ce4a4eadf265d4481ad7f6864edf1d7e3084c6ee32cbbfe454bb2946bfe"} +{"family": "injection", "fold": "outer-4", "group": "dolevf/damn-vulnerable-graphql-application", "label": 0, "pair": "5b836889b4da4921", "unit": "caeed7f097a3311fe2e84efd372fe4b4c140f63a80a9f07ba0b61966dbab15e0"} +{"family": "access_control", "fold": "outer-2", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "c2a2c2d85114a97f892fb1e5500fbd98622457a8c46451e1f9e958ab08a40318"} +{"family": "injection", "fold": "outer-2", "group": "erev0s/vampi", "label": 1, "pair": "", "unit": "29d8be8206c88ba7efd86b56a7dcf8b655e33b4a792c2d18b028c92c6252daab"} +{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 0, "pair": "f96f0fb245948915", "unit": "0954e6931373915c204b9ec8d2b8ae6297a95667bfd41aadf9035625ad93967f"} +{"family": "output_encoding", "fold": "outer-2", "group": "etaminstudio/prosemirror_to_html", "label": 1, "pair": "f96f0fb245948915", "unit": "5e09cdae2504e834d591b6f92a808427f07550364b5419899f7e6f76084fc377"} +{"family": "output_encoding", "fold": "outer-4", "group": "ether/etherpad", "label": 0, "pair": "bf11f138fe39675e", "unit": "56f0d0b156992af56141d6368fdb0c5376fc7a44a03991d754f97202b9d77008"} +{"family": "output_encoding", "fold": "outer-4", "group": "ether/etherpad", "label": 1, "pair": "bf11f138fe39675e", "unit": "8271af5017894167556893da2070188060d06eebc87f27db45c46234c359631d"} +{"family": "path", "fold": "outer-2", "group": "fastify/fastify-static", "label": 0, "pair": "9d937c2dffb4aadc", "unit": "01ac12bb06e660d0aeef8b1e26dddc5c99d52cc333f7af6d1981eee2818e64fd"} +{"family": "path", "fold": "outer-2", "group": "fastify/fastify-static", "label": 1, "pair": "9d937c2dffb4aadc", "unit": "9612edf0f927048335e83e9925f01be9aea9b772bdbdc568ad9cc0a17055662a"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 1, "pair": "d4f648048e45f2ea", "unit": "08807c8110b8c1b8635a6883c229fb91f2b9566db8b6c252448a0db7b2ddd47e"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "feathersjs/feathers", "label": 0, "pair": "d4f648048e45f2ea", "unit": "10869c931053bf77ecaa98a47f56dea27d643d1abed2a4758bcf48a78547ccc9"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "662e1f36c230d7f1", "unit": "0b73332b25413902c26d5b5dee6767465eed99ff71a814582988ffe0d8ec1a30"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "8c8e46d0c7d361b784baff9d70f38744b2805ef1ee0aaacaa616bc20a2c6f32d"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "aca938682c196fd4c85b39b89eeedb97c35f6a5a017651fe9c03c5b727cab7ea"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 0, "pair": "", "unit": "cec00d1509a129364da6ab8537fb3f433fd847e26d073c90311ccdaf7a1b39e7"} +{"family": "injection", "fold": "outer-1", "group": "fecommunity/reactpress", "label": 1, "pair": "662e1f36c230d7f1", "unit": "e0f58a7ac5627de3e7b6cfe5add41d18b0f64b998c1cc788d42eb2cdbbc3fe41"} +{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 0, "pair": "2c251a645094f389", "unit": "0d41cc634771048f72d07c4aa9feaed669d0bfe3844d61dcdc525a0f899617e2"} +{"family": "access_control", "fold": "outer-0", "group": "filebrowser/filebrowser", "label": 1, "pair": "2c251a645094f389", "unit": "ca8288d0b8b94dc6b7e4c681936ea610d295fe949cba5f64f09b6de134ef9875"} +{"family": "access_control", "fold": "outer-2", "group": "forgekeep/nebula-mesh", "label": 1, "pair": "2ac5ec5011615c9b", "unit": "2f427f403af3cb58d776437736c2389805543413fdb91aad98979158c6d00de3"} +{"family": "access_control", "fold": "outer-2", "group": "forgekeep/nebula-mesh", "label": 0, "pair": "2ac5ec5011615c9b", "unit": "cddc8c5bed0841b80e7b20d0b378e858f8e4ac37fd7b890ce76611d989857f92"} +{"family": "injection", "fold": "outer-2", "group": "frangoteam/fuxa", "label": 1, "pair": "", "unit": "0c7830c2912ff285855d8a3c8e9b47db4ca4af840baa66d4b5d3f4422f46f95d"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "5383a8c2a78fabd3d17a5b3021125953b5ebbee61f8900c30d016c56b1a3bad6"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "360d8bfd2f116e98", "unit": "7c1989a4ec3622bdaebcf1161de850f22cfed041cb38830ca1ded4fa99d40466"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "ec8c706cc2b20f36", "unit": "848688dcd0fe8d46e6762cc8aee1d82e8fad8a2a044c94b547aea11049c85bfc"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "ec8c706cc2b20f36", "unit": "983cc9231f91baee4e2d626193f3b88aa5b09907f6faab870949bdc9c336b816"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "e4ad86416d5c6668", "unit": "98ff1dff2afe80709b8e61a9c95cfeea12b2110c13b7120478268b4762e2c992"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "360d8bfd2f116e98", "unit": "a1a825d23e34757c8e961e43ae791253e62682e130acdb846b58118588006a98"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 0, "pair": "", "unit": "a7fa4207508b3989baabc7a1f0ff2ea5229b366c1cb106e7fc945134b8b106c8"} +{"family": "injection", "fold": "outer-4", "group": "froxlor/froxlor", "label": 1, "pair": "e4ad86416d5c6668", "unit": "ef5c53f3114d0057e694df13dc07cbcb532c3b0d59150eda269b87b021d7a6e8"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "getgrav/grav", "label": 1, "pair": "9ae575048504c690", "unit": "176acf077fd58c3e6f940753179ffa3e8316661f355d94acb5e0345b51925ca4"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "getgrav/grav", "label": 0, "pair": "9ae575048504c690", "unit": "228b34ebc2090b5239c7821d28366ddc663e7118d015a79b45dca6618d5490fb"} +{"family": "output_encoding", "fold": "outer-0", "group": "gofiber/template", "label": 0, "pair": "a3fdf9ce3441ec86", "unit": "653236018929ab8534c27704a20ec5d358047bc66e451620d2ac32081c91d333"} +{"family": "output_encoding", "fold": "outer-0", "group": "gofiber/template", "label": 1, "pair": "a3fdf9ce3441ec86", "unit": "b70ca2c43f5929e7b885bd728349e721fd85d7605e2f722a7e8cfe0f0d477b0c"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "groundhoggwp/groundhogg", "label": 1, "pair": "", "unit": "92400e83890882a66ffdea1a405e2354a195442e176d85fb707b0442c44d45ca"} +{"family": "path", "fold": "outer-1", "group": "gtsteffaniak/filebrowser", "label": 0, "pair": "f9dcf3a0fb0f0a14", "unit": "7ca6dd4f673976b000a657f885af3463740419ce41e14f972af0c5483cb75a7d"} +{"family": "path", "fold": "outer-1", "group": "gtsteffaniak/filebrowser", "label": 1, "pair": "f9dcf3a0fb0f0a14", "unit": "86c1db3a5e9f843d6546146449171e0378ac61d6c8f44c6dd72fd379cdbb188a"} +{"family": "injection", "fold": "outer-4", "group": "handsontable/formula-parser", "label": 0, "pair": "6144baeb491f4a56", "unit": "f5c22ce104e9593e5272b6e8b3c6f3b61fa4578552ccea80e6021ee90bdb3ffc"} +{"family": "injection", "fold": "outer-4", "group": "handsontable/formula-parser", "label": 1, "pair": "6144baeb491f4a56", "unit": "f687e7e95dbe47f602a1e0002a44f94fd48e5b1c33f77ecebc6c2d814fcbf450"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "heartcombo/devise", "label": 0, "pair": "2ef4471c66e5e4ab", "unit": "ad7c8e368fe1699a044e058d5c10b41bd9fc0f91c3f3c384c5dac40af39e71f6"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "heartcombo/devise", "label": 1, "pair": "2ef4471c66e5e4ab", "unit": "d888baf7194bbe7158cb3932f58b47a9a235351d388d10df82a130ddd54c2d46"} +{"family": "output_encoding", "fold": "outer-1", "group": "honojs/hono", "label": 1, "pair": "5628c54152216646", "unit": "47d081c9f6060218967902f694e61c88f64f57c65bec6fc3073afeb0b77d117c"} +{"family": "output_encoding", "fold": "outer-1", "group": "honojs/hono", "label": 0, "pair": "5628c54152216646", "unit": "e1297bccf9c3606e0b41d762a69ff5d0a33ae4f7f1506071af6b9cb4a177ff56"} +{"family": "path", "fold": "outer-2", "group": "hs-web/hsweb-framework", "label": 1, "pair": "e52aa814c36ace1f", "unit": "2d6d74364f1982e91f27f1c995b8699df1fc8f8dce6e1e2cf274bf01b6e9494e"} +{"family": "path", "fold": "outer-2", "group": "hs-web/hsweb-framework", "label": 0, "pair": "e52aa814c36ace1f", "unit": "4727471fea2cf0a92d43b4b1503f1339a7dde87f872ba8064dacbf855c57705b"} +{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 0, "pair": "6548e1527e82bd99", "unit": "5542613f6d2eb9421b8213b8bb864894f0ecf9fe64945681019230ef0d75cfaa"} +{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 0, "pair": "", "unit": "6c2adee34e9e8384535ec1073eda60e2ab528ac77c36747802c303e1f5e9ffe8"} +{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "7ce70e9b20107cc7ebb87d173da765fc53ffd268e521749a06a9a1f38d746845"} +{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 1, "pair": "6548e1527e82bd99", "unit": "c3e54975464323e58ec96ee4d83e7bc08baf94553d553a8dc3fb28cd9c70ff21"} +{"family": "injection", "fold": "outer-1", "group": "ibm/mcp-context-forge", "label": 1, "pair": "", "unit": "d3173c1e43eb6402a71ea2dec1592913dc078a2194ba1c1d509ab129a3180f6f"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "internlm/lmdeploy", "label": 1, "pair": "0924befc49c7bb18", "unit": "e7e4872db40f0420547ed622fbdd7a072b64a7ba466d1ccfff68e691cee6ad99"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "internlm/lmdeploy", "label": 0, "pair": "0924befc49c7bb18", "unit": "f7d95f43fb2df20ebc4c37fceabdd00ffd8159f404ca384481f1764252382388"} +{"family": "path", "fold": "outer-1", "group": "jarofghosts/glance", "label": 1, "pair": "772e18821701d60f", "unit": "89a0496c7663bbfd67d660c9f481c12420da2dd1bdb29dc079735ee7a0a49626"} +{"family": "path", "fold": "outer-1", "group": "jarofghosts/glance", "label": 0, "pair": "772e18821701d60f", "unit": "d3241ceaa50a7d23a4650e6a2a55372b08ec96b35b0bede844bc40e16e818ebc"} +{"family": "injection", "fold": "outer-0", "group": "jas-/node-libnmap", "label": 1, "pair": "250de1302c7e541d", "unit": "5d03535f2df2acdbc319d9e6b20da65d51f3e810b80f3c8f89b5074a971a3f1c"} +{"family": "injection", "fold": "outer-0", "group": "jas-/node-libnmap", "label": 0, "pair": "250de1302c7e541d", "unit": "78f977cfc096e34b4eada7f798ad2973ef45728d0d01bfab6ac9e2a3257827ef"} +{"family": "injection", "fold": "outer-0", "group": "jashkenas/underscore", "label": 1, "pair": "c10bfb006f2da55a", "unit": "1ea4b86dbd84581b1d7b7cabbab9f28c8b213416a60778879666ff42d23fd7f1"} +{"family": "injection", "fold": "outer-0", "group": "jashkenas/underscore", "label": 0, "pair": "c10bfb006f2da55a", "unit": "7647021f742da24c660b2d407886771786e957076c4cf336def560a570dd8778"} +{"family": "access_control", "fold": "outer-0", "group": "jellyfin/jellyfin", "label": 1, "pair": "a73421efb106ab0a", "unit": "1f6f207a758582de0c0e6b331b19fe119abe18727d45ccbbe09aabb795f3a292"} +{"family": "access_control", "fold": "outer-0", "group": "jellyfin/jellyfin", "label": 0, "pair": "a73421efb106ab0a", "unit": "32fbdad39eaf5177db4ad0a11d53e785b30bc7b2376ad9d460f29a08429bd0c7"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "jenkinsci/cas-plugin", "label": 0, "pair": "0458e3fd444409f9", "unit": "b521216bb4106738b3571ad3902d08f6230aabd88cf3a8f9f26f1c8ec01304cb"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "jenkinsci/cas-plugin", "label": 1, "pair": "0458e3fd444409f9", "unit": "c87d2871907798fa2e73d31381e9d63089b0388033f096d247373fec07c2cb36"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "jenkinsci/google-login-plugin", "label": 0, "pair": "6bdccabcf5b9fdc9", "unit": "3d6ddca5eb264d88411b86755909f1d2abddd09c129cabd54c96ff26b4105316"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "jenkinsci/google-login-plugin", "label": 1, "pair": "6bdccabcf5b9fdc9", "unit": "e055b898802279bc63204cc601e29f7f1018efad7228cab0cce00ea5866ecc7b"} +{"family": "access_control", "fold": "outer-2", "group": "jenkinsci/mcp-server-plugin", "label": 1, "pair": "91bc832e70680c92", "unit": "1060c63ff0fae3af567448f3d4c60d9c159b61ccc358964243eb152a9ac38865"} +{"family": "access_control", "fold": "outer-2", "group": "jenkinsci/mcp-server-plugin", "label": 0, "pair": "91bc832e70680c92", "unit": "369dffd1e730d7997813adfbebf1481cc68987cae0d0085e47b91a26d5330cf8"} +{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 1, "pair": "a4144258dffb3b8d", "unit": "7ab301e1614979f44c353e665540582493f78ef4bfc41209562c19b900710f8c"} +{"family": "path", "fold": "outer-4", "group": "jknack/handlebars.java", "label": 0, "pair": "a4144258dffb3b8d", "unit": "ea95e8387dff32ded9dc7b04c11a76a545b4e2ea6db0076b9b13971a59904bb3"} +{"family": "deserialization", "fold": "outer-2", "group": "jmespath/jmespath.rb", "label": 1, "pair": "858330e01d3cf9bb", "unit": "033cda2b7cffa5a5fa6abf0b7f2cd27ba78d916eb3cab8c79decccd8776f4a3f"} +{"family": "deserialization", "fold": "outer-2", "group": "jmespath/jmespath.rb", "label": 0, "pair": "858330e01d3cf9bb", "unit": "d44c4555a5ae19aa17b468ec5cff56b33a0b221df52e04d8abcb656399499dee"} +{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 1, "pair": "c6a49339d4f0c798", "unit": "dc2e4cd5bd963643702d2b832ae170dd25744162e4076e36687256a09e69d089"} +{"family": "deserialization", "fold": "outer-2", "group": "jooby-project/jooby", "label": 0, "pair": "c6a49339d4f0c798", "unit": "f83cd137b85e651ca5299c5a42176b797ed4964cbef31418c96d63c52fdb65f5"} +{"family": "access_control", "fold": "outer-1", "group": "jsonbored/gittensory", "label": 1, "pair": "5ec37c0f475001ca", "unit": "5d8175fb21a4649bdc8730358e9b87adb90b2aafc4da5fee203829663e4f46a0"} +{"family": "access_control", "fold": "outer-1", "group": "jsonbored/gittensory", "label": 0, "pair": "5ec37c0f475001ca", "unit": "af845e9e332555096dd350401977b2f07015433f3db8a040891d9470f3f9e355"} +{"family": "deserialization", "fold": "outer-0", "group": "kevin1024/vcrpy", "label": 1, "pair": "c74525e844650de2", "unit": "2e444ad34ee5f47d73f0cfad0a10c70363337b4ddf9067ecd1b8072d8232652c"} +{"family": "deserialization", "fold": "outer-0", "group": "kevin1024/vcrpy", "label": 0, "pair": "c74525e844650de2", "unit": "c492d325f5ff82b5f817aa4be1440cec8595054dd0719e07a58ac5501b2ae7eb"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "koajs/koa", "label": 1, "pair": "111a5a80a3abc2c9", "unit": "c7ae403742450deb2bd051ad1231121d2117ba56fa72a038c82d58d207cb3870"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "koajs/koa", "label": 0, "pair": "111a5a80a3abc2c9", "unit": "debe34831afca6f8220b9952036a2d4570babdb5e321a2f8dbaec661a016bd50"} +{"family": "access_control", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "9950b85412b021ebabb11eea4c490a660ad380d14c673a87c3fb568ba4bcfac9"} +{"family": "path", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "656df5ff8e69dcdb1f9ba1ed52d734966eb29773bc745587483a355743b8eb5a"} +{"family": "access_control", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "91652a4e776fbce70cde9e4a99c6da5e17da7f8d6d0404867c61eec04161b6cf"} +{"family": "path", "fold": "outer-3", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e53bb3a217dc4737e8dd03671de4f6a895599024070a985a268c65a034295e7c"} +{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "3d7aba22444811a337abcad0e4fc62990bb704d3a10e72d893342fa94699026b"} +{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "5f18a3026a51f7140b7aaa3aa8563e8b6fdda6065d95e19e25b1dac4b9ce0e00"} +{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "0fccc306ce80cb4cbfdcde0a2e56d637bcefa9dd30423d6c54a4f77860afe740"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-logistics-dispatch-fastapi", "label": 1, "pair": "", "unit": "1e2eaa45661345f1b6759d968d7696dd0d96d6fba1350a00e4d0a2ebaaa2e391"} +{"family": "output_encoding", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "8b9a8ff0e32a327df75eb8a1811464d5f34690079d5930139dd917063bdd2809"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "245d0e939b7c1ccf098c798492009e3a73ea17400fa613dd865cd369d2e7f383"} +{"family": "access_control", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "653c48df493f31c2d01c973d1e379a180fbe6a6edc83a500bc62509112175f41"} +{"family": "path", "fold": "outer-4", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "3719fe21d45b50b35458bcd2e263e4baee67adbd3c98adf0eaab6bc6183c5651"} +{"family": "output_encoding", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "1209a6cc8c1548ed83d4a9a09d6a27f706a23e69cf0c261fc5e3a27536a83306"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-claude-code-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "362c90495dc468bb732c71f62d71ec308854161de31ad18d2a870056d947ddb6"} +{"family": "access_control", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "b52e53fec9707b2b442c4dc7a41d376cf76946d2c45c2eb600c9477fc6baebaa"} +{"family": "path", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "477e3b90b92e622888dbe0e9d3a23b31271ddfa3104a5ab326eb9e9e765aa01d"} +{"family": "output_encoding", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "660c63c4a73bfe64088ae3249e40d9c5d69a6f8a505ff671217c5708a345c5b0"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "4d60c386c801bedb61c1461fced29a3cfb50b09d6435e8617c60398343719d93"} +{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "f1fddcc1354a29bef55610ee2b73eaa440192f2f18a0d22d08057b9d1e1b61b8"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "6d9886c38221b2f9aacf83749218ac5ebc354a6fcc4170e6c78bedb92b88296a"} +{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "5550fec4a81c4c9cb22d7312d8a63e1fb2b03c4aed0c1f316d44c1eb27511bac"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8ff87dd04140f0137ea1891f220b1276156ce10a58be5a49d0c8e07be531ffb2"} +{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "c6c9bde6630678377d250701073c5c223c7892e6ff8f2f8e0293dea22c5b5e7e"} +{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "b9b15958973c83f8d7b58372ab5ba58eb191c400125579d5ae8c5b52d5d1368e"} +{"family": "output_encoding", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "b95ec12e4e126ad558e944e2c260116ed6a63f5f3ab97fb5cbcdca75a931ad73"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "38a5545ac9ef67e15db720d1396a2d5915f9d06d6f4772012e5bc95f33770376"} +{"family": "access_control", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "c8bbe9be91c976be97b7ca387048a7d4537d9f6fcc6cbad0a541d1080629d5a6"} +{"family": "path", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-high-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "ffacd53f1bd75155e63b318fb9cba27ec03e754841e1eb39d1d4fa0b2c3dfca2"} +{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "8fd920175fc43b91b4397acffacb10ab8f7fd0ee38bcf2d2f3ed06a3fad0dfe9"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "86c9598ebe3dcda261a69ae545513f6ce73101f3980066e4506ece99bebea84f"} +{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "e5190fcc57505116f79a31c51aed69f8a1eea13a8ae3e7265f44c72418277ef9"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-codex-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "2b807ff54e6c47e941124c16d0c27c691cef54f2203060a3a6ce812d478c5261"} +{"family": "deserialization", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "401aefb4b4791301ace2d770d321c6b26743b0aab7b96f8b410254c704be6d2e"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-codex-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "22d186c93498e21418b6c1d0547a73374b13041d8afc89a80d9540fa658b6b42"} +{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "dbfb6df14fe37f8c0efda5f16643297fb0550dfc5ad5805d4e4b844d967773b2"} +{"family": "output_encoding", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "f0da8cfe4ac1e586aa7903d72e67b44500bbd5306d72b4b342d10613efc69222"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-codex-seeded-v2-support-desk-fastapi", "label": 1, "pair": "", "unit": "2b8fd73943f456da266434470333a7775f55d9a41953e97a000fdaa3257b6a30"} +{"family": "deserialization", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "a612091f452e60f28f44eefec641175170dc5db04456a39a099cbb13c50b4d2c"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-crm-saas-django", "label": 1, "pair": "", "unit": "4c6600facd83bbf234a32d9e28ac494fb516df6e3d9e9dc28d720ff35252d067"} +{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "fb0ea8cab8c9430d22d1b0fdd65a69146fa95fbff94e720b4861b96538d992c6"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-education-lms-django", "label": 1, "pair": "", "unit": "e81f91a56058ac1473579767abea57064c53d7d007797a854314814e3c35ee74"} +{"family": "access_control", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "5851ecd96a449f98b05f243bfda0a7cf3af8612b27f53c414ae1f4ea67242a7f"} +{"family": "path", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-fintech-lending-fastapi", "label": 1, "pair": "", "unit": "328621ef2a94632e5d37ea6267d17b766233384cde35c07e29a4eb672d40ba28"} +{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "04b2eae2128608dad460ba8275142a41a44e32c4243a29e2a6fb2c9ebf3d8f1a"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-healthcare-clinic-django", "label": 1, "pair": "", "unit": "33337002ff5f5a189610b042522ceea254c9d107c1f24771b4f616a0f96f636e"} +{"family": "deserialization", "fold": "outer-4", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "02c09a783cbf058856fc2b448406f3ffa6fbfbacf9b70f4e5e22fd741a5c7699"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-hr-payroll-django", "label": 1, "pair": "", "unit": "956260164cb21a47c499e5598491031ef303418943f5096f225a87162b588574"} +{"family": "deserialization", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "8bd8c5aed69eae3019cc2d3654b36bdf2b31435e6a020a389cd103696a2f874c"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-legal-case-django", "label": 1, "pair": "", "unit": "1d8397ef4a9fde7b48e2e1e871bba0f960a802cd7823ecfc10e9dfaddb955ccc"} +{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "c9db92476032eb4296b950a1eff8fa4aaa26f11542a01a5cf10c7b2e32dc6433"} +{"family": "output_encoding", "fold": "outer-3", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-marketplace-commerce-fastapi", "label": 1, "pair": "", "unit": "d3e0d0f9b0f4b723080989b5486395cda0a5b186a674022ea70e7fa7ab561323"} +{"family": "deserialization", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "44ff3b3e61c02744830c4abea872821ca7e0f94950c082740c5c48ef13604e94"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "kolega-ai-dev/vc-kimi-code-seeded-v2-property-management-fastapi", "label": 1, "pair": "", "unit": "4103e979a99233c3e71fdfe9f1cdcbab34a2c60b5351b75c5ff5dfc287b962d0"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kozea/weasyprint", "label": 1, "pair": "6768bfae38c6859d", "unit": "f376519889a00950f5af67eb15c639306ea70da7235d3be4a94dbc19cb096cf5"} +{"family": "untrusted_destination", "fold": "outer-0", "group": "kozea/weasyprint", "label": 0, "pair": "6768bfae38c6859d", "unit": "f3fea4e334f49ef6fe6b17121fef553eb8e33a9ffa13e6d6cbc4c448da996a7c"} +{"family": "access_control", "fold": "outer-1", "group": "kubev2v/migration-planner", "label": 1, "pair": "d670253fa209671c", "unit": "1aeebe2e440bdd70525c31f93180f33be93f820db5ba46a2e59c371ef83ee794"} +{"family": "access_control", "fold": "outer-1", "group": "kubev2v/migration-planner", "label": 0, "pair": "d670253fa209671c", "unit": "b11c8a8a24d3809c0507228568fe7cc05ca0a22ed28cb19019adb13cb619d63f"} +{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 0, "pair": "33260f3e6c8ea6d0", "unit": "7ca409c7dbc1c38ff005624e212efc6d424cb421e27b482dc8dc0d6881f4c11a"} +{"family": "injection", "fold": "outer-0", "group": "kylefarris/clamscan", "label": 1, "pair": "33260f3e6c8ea6d0", "unit": "fc364d7ed77c7c0508da00e5ba847ec0792f3bc373d8b4297747189ef42d1ae0"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "", "unit": "6cce78d7569ad8db7ca0a61d7078198f666502d26ea6c0afb03173a8a4bb16ef"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "82047c0513e43a83", "unit": "6d2dbed1ab1c9d8ac259dd4227be69641fa59d52e816a8dc547ade01d0b9b0c4"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "98ee23cd6bd31cd3", "unit": "ab4bb39be535d70cd95e9035bb958bfc7418576a6ff40a47551894dd140317bb"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "98ee23cd6bd31cd3", "unit": "b70068cbc646bd01d3e10986e002f648a74e9d798900dc59cf3039b749c95c18"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "ff168d95d8d9496e", "unit": "c6ec9478c125ed780d7013c2807f2cec1878f44253667dfed319c28cbfd17d4d"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "b1bfef46dcd3b9a5", "unit": "cd4d095d8380a2fb7ca02fc6abdbae5249cc06455db2929875533f4aa8b886dc"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "82047c0513e43a83", "unit": "d1bade217be8d345cf5d0f0442ee31924187acbb98779dada95274a77462f696"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 0, "pair": "ff168d95d8d9496e", "unit": "d561db250b892b91feed7ae877900e1e25963444f73c7699576e010f06da73cf"} +{"family": "injection", "fold": "outer-0", "group": "learnpress/learnpress", "label": 1, "pair": "b1bfef46dcd3b9a5", "unit": "ed78fd5a7ab30bcd6e8ddd27c706a19cfd89750de4fbf81c0b567946eb94db83"} +{"family": "output_encoding", "fold": "outer-2", "group": "lepture/mistune", "label": 0, "pair": "25f75d77a669b54e", "unit": "0f2825568461562f2a915a4f16ef8a4931290fd20a9801470aeb12677e66e3f1"} +{"family": "output_encoding", "fold": "outer-2", "group": "lepture/mistune", "label": 1, "pair": "25f75d77a669b54e", "unit": "bf22bb44bcff4a5baf5d550c9d3e83e5f3ca607475fb51deaf4fb6fccff3d73a"} +{"family": "output_encoding", "fold": "outer-4", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "cd80750447cf7af67ad809279718d98106c71705fef9ff7254728bfff4ece5b1"} +{"family": "path", "fold": "outer-4", "group": "linuxuser255/vulnerable_python_apps", "label": 1, "pair": "", "unit": "2ce605ab5ce7ad3a1fb983cd6bddf3924e5da3010d2252a0a4640aab4e5921a0"} +{"family": "deserialization", "fold": "outer-1", "group": "litedb-org/litedb", "label": 1, "pair": "b0a899116dc36b85", "unit": "510ad48af03ba4a1a2a06dd254b135311dddb62d9fe5a2e8e8b12627586e6b13"} +{"family": "deserialization", "fold": "outer-1", "group": "litedb-org/litedb", "label": 0, "pair": "b0a899116dc36b85", "unit": "b414718a8a5f79739b88db24da6ad9da8e48bcb6b5e2dc99158848cdaa226767"} +{"family": "deserialization", "fold": "outer-4", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "8e22ee30441b20484090659b0c97423c711f5d10219041bf3a399623fc056a07"} +{"family": "injection", "fold": "outer-4", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "0f9070df9dc334f981cb01008c260b8bff78dbdeb48ce672830ed95caa8406fb"} +{"family": "injection", "fold": "outer-4", "group": "manuelz120/extremely-vulnerable-flask-app", "label": 1, "pair": "", "unit": "5e352f797b1422b0df477bce4730a9ce33cbd0837c3d63da94e4d8801a9a4b64"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 1, "pair": "dadccb128ac0e120", "unit": "505097494da5939404ad7d5b4af2b1c4a10efe94048084492d6ce60ebaa04c97"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "dadccb128ac0e120", "unit": "84ff8ca2d98a1b5d1ca4bfaabe7d19e03a396042b228c8d18d1185a178389702"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "91e261c54bd2c3c3a9ac04e48894ecabaca5b13e7db4fbd5578f9d266098117a"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "a308dfd59bc398824e2454bf6cf6cfd5b7880e5b948a82424f14753d2b6fcdbe"} +{"family": "untrusted_destination", "fold": "outer-2", "group": "medplum/medplum", "label": 0, "pair": "", "unit": "ba24eb554458bc97314ed561a7b2f4a4e643dde1205c32de4c8fcdd8c46e575b"} +{"family": "output_encoding", "fold": "outer-0", "group": "mganss/htmlsanitizer", "label": 0, "pair": "24d8f1840b67fbe5", "unit": "7ffa4a229a62266788fbf64890f81f20358b3eb8c35d3ab87d0c339d5bb4b507"} +{"family": "output_encoding", "fold": "outer-0", "group": "mganss/htmlsanitizer", "label": 1, "pair": "24d8f1840b67fbe5", "unit": "d19493ed7941e1cd25a76dc50504d97398770d07ebc302dd66d83441e946c468"} +{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 0, "pair": "9f8a60eb48d74bc8", "unit": "80fce34f2377731e3de4355c7ea33bb20fc2fc00a5f388565b23018c7a3ed008"} +{"family": "path", "fold": "outer-3", "group": "mguinness/elfinder.aspnet", "label": 1, "pair": "9f8a60eb48d74bc8", "unit": "b68e62722a327c9d5ce6774eb96527451729410768efc0b5c7ba604ec468052d"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "miniflux/v2", "label": 0, "pair": "997f9d2efc32302d", "unit": "b587bb499945a9f5f0749eb4d57cc1027cfd21c8292bd32be4d4e1cc1b1db261"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "miniflux/v2", "label": 1, "pair": "997f9d2efc32302d", "unit": "f8d9c9a7c12797b3caf042e6bfdb89a5072c4eae1fd36be6196f9fa326cdf257"} +{"family": "output_encoding", "fold": "outer-4", "group": "mixxorz/slippers", "label": 0, "pair": "8e6b4b4e28981590", "unit": "3cce3e8f9ff3f1d261f9b96c844c1e4da07bae72d5393897b3a929be25372edc"} +{"family": "output_encoding", "fold": "outer-4", "group": "mixxorz/slippers", "label": 1, "pair": "8e6b4b4e28981590", "unit": "b105689ac24ed334294c434a77bb6d56d5d598830ffe8a5d60ab3be5290b47ad"} +{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "0a36cd5177f3b2c20b433db6bfffe5ec5b7e74cacc34a86ac581f7a0a559432a"} +{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "e7fc97d83013c9ff", "unit": "5ab444e64297fc19065478104c7ebdabe23cc415f22adafd0cf82c98e4c5cae3"} +{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 0, "pair": "e7fc97d83013c9ff", "unit": "6b462ea30c1afda572600a10324249223932d36a032eacafac8c837bccbdce87"} +{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "84d6a4202a082e0d", "unit": "8175082f40d8d61d44b54579c84a808e25b89110be5d8fd82feedf7dd5e0d5ab"} +{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "8abefca43ee4ae2c797033da31f85c17bfa7be6b35413d70b0216415c5db9a09"} +{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "91d213946859f33d3f243c43aa8285cc6e8ecb3db1de938f32af775a70607f8e"} +{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "a244d9f010ddb43a09e82363dfabb0e406d411fb5de08ab7d6c9ee367f2a15c3"} +{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 1, "pair": "", "unit": "d27d174d6e8e5f1682a1c42723705e89f0d18c95ecedd0567e313daa9a0aae44"} +{"family": "injection", "fold": "outer-3", "group": "mongo-express/mongo-express", "label": 0, "pair": "84d6a4202a082e0d", "unit": "f45608ed47fa11ca25093641f6021ec06076c18676abac9b809f31b3b7713de8"} +{"family": "access_control", "fold": "outer-4", "group": "moquette-io/moquette", "label": 0, "pair": "55d1ce4f29dd1583", "unit": "a877ce3462da316bcdd82ce9db1d9887e887eaa83157922edb1663413edae1d6"} +{"family": "access_control", "fold": "outer-4", "group": "moquette-io/moquette", "label": 1, "pair": "55d1ce4f29dd1583", "unit": "aee67be0ba31f99c7a24faa1bad6178c6d37361a308cf5aa10138e6ce4bbcfeb"} +{"family": "path", "fold": "outer-0", "group": "naorsabag/openhop", "label": 1, "pair": "16e529fe8c5486b2", "unit": "1d21acd0171726775d3da4589b660645fcd43cc026042a2b27e0e0e8374eceb7"} +{"family": "path", "fold": "outer-0", "group": "naorsabag/openhop", "label": 0, "pair": "16e529fe8c5486b2", "unit": "f8e004b67ac98dc3392ea4115051e6bd10ce22eb638a762f0f16e1d4200bf129"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "nuxt-modules/og-image", "label": 1, "pair": "ef603436079b0005", "unit": "77f843b5044bb5da935df4b2620bf2abe9e7dceb3147439e883927112eaec22f"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "nuxt-modules/og-image", "label": 0, "pair": "ef603436079b0005", "unit": "f2ce95f0e44e73986784b63a702ffa4035c423056cac3667a63140a7e8d795c2"} +{"family": "deserialization", "fold": "outer-3", "group": "opensearch-project/opensearch-ruby", "label": 1, "pair": "1945f28a3fc5bd55", "unit": "64c0b7ccad09192eeaf590befe0de0087d5b48a65a4f6cef41415f3f54a8f536"} +{"family": "deserialization", "fold": "outer-3", "group": "opensearch-project/opensearch-ruby", "label": 0, "pair": "1945f28a3fc5bd55", "unit": "b55cd6759d4b7861e589052aa534bd0554586db46a329ed3beed7244d579e8c4"} +{"family": "injection", "fold": "outer-3", "group": "openssl/openssl", "label": 0, "pair": "1c9e47835232f644", "unit": "90e220d3540b88dd9efc8e66c8ab519d4cc78805f16bed585b795c78cfbb4d58"} +{"family": "injection", "fold": "outer-3", "group": "openssl/openssl", "label": 1, "pair": "1c9e47835232f644", "unit": "b5a185b05235f70f6cf27e2adba919f4d8393dbdf2d99de095548a21946aeaf3"} +{"family": "access_control", "fold": "outer-3", "group": "openwisp/openwisp-ipam", "label": 1, "pair": "9b2b34cf62a76ea3", "unit": "91a3984e47f17512db8523e04f863525c67db18631d8f11776cf61ca7221c91b"} +{"family": "access_control", "fold": "outer-3", "group": "openwisp/openwisp-ipam", "label": 0, "pair": "9b2b34cf62a76ea3", "unit": "d91f1e14282a0c3c8da57c6935a1625e881374bd205b5baa9e4a341d9109ae21"} +{"family": "access_control", "fold": "outer-3", "group": "oqtane/oqtane.framework", "label": 0, "pair": "a20cdf9dd208720d", "unit": "18a26df68e4b81d411cfbcc8155a7e6f382753d6e1ad95eece7ed24fa8c44324"} +{"family": "access_control", "fold": "outer-3", "group": "oqtane/oqtane.framework", "label": 1, "pair": "a20cdf9dd208720d", "unit": "e6fb74441fb585d7ef5855317fba795ef0f0a0e6eae9970a49556ce3070c3254"} +{"family": "path", "fold": "outer-4", "group": "owen2345/camaleon-cms", "label": 1, "pair": "d6930fe7e752b7ec", "unit": "06f0341dfbf1aa9504083737df5d9be3cedfd4e3a198799d8075a29990776800"} +{"family": "path", "fold": "outer-4", "group": "owen2345/camaleon-cms", "label": 0, "pair": "d6930fe7e752b7ec", "unit": "30e2cd8bd381d7568b78eaf982f562d278a6a8d942d5282466532c1441bdf0ad"} +{"family": "access_control", "fold": "outer-4", "group": "perses/perses", "label": 0, "pair": "2d482951d7380c4f", "unit": "5c1a721da6f85d1d5dfc8cbe5e9809b093a09c059e5abeebed2cbbee55fbd498"} +{"family": "access_control", "fold": "outer-4", "group": "perses/perses", "label": 1, "pair": "2d482951d7380c4f", "unit": "6cabd1c4b5aa94cafdd07c0c6a3f9abfdc03761ddc5b0a2bebc72a508d020b8b"} +{"family": "injection", "fold": "outer-2", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "", "unit": "2300c097374b9098671dcbcd42d02d08172e0f3064a8323223d0fe79a6106aab"} +{"family": "injection", "fold": "outer-2", "group": "pgadmin-org/pgadmin4", "label": 1, "pair": "0664148b0b192456", "unit": "d702617aaf4d19811baaa18386a3c8b0eb35cd3fd72e8d90611d34850e990858"} +{"family": "injection", "fold": "outer-2", "group": "pgadmin-org/pgadmin4", "label": 0, "pair": "0664148b0b192456", "unit": "f5ed13d811c5b16d46cbd4a3b9edb8f7d2db49acbd49d95f39f33eaab38f58f5"} +{"family": "access_control", "fold": "outer-1", "group": "publify/publify", "label": 0, "pair": "1972ea9f72b178b6", "unit": "3ddabb356d78e7c65ec8f88b6da8e4fcabc055ae73f2f6d92c50efddc5d4fcbe"} +{"family": "access_control", "fold": "outer-1", "group": "publify/publify", "label": 1, "pair": "1972ea9f72b178b6", "unit": "6875dd5e9c0ff8a1b693d29a109cb2e62e6ff4081df40868aca70e4770476757"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "04aa777130c02a06", "unit": "16aa43c87956c44a4b115245140eefed704095d37cec0cc6410377403cd2fa7a"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "23744137e0485a36f686f5f1b5666780fbed3ca522e8c6d35687a391ef504498"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "67565d99fe28c68f", "unit": "2692316792e6f02fb51578cf30d71bba6c73bf5420c36f709681c8eaa69807d4"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "2e36f73002c416a68d077bffb0335d82346c7507f2478353aeac005fce5a9cbc"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "04aa777130c02a06", "unit": "67721f67055eb6fed955ff22bfe4060344662eccb369431835a2116ad120c4f8"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "8fc1c96fa6faa4d3966ecc42876189b18ddab779d58f47cdf28a1155f48a6ac4"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 1, "pair": "67565d99fe28c68f", "unit": "ab535098a8e709b022afee1da15dd1fc371d389056938ee4fa4971b2966c823a"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "d46f94235bc462360097e7a22e1d442d95f049306631489a4d8b25f647a95bbe"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "edd6dd4364442f66384d14e9317ef2f6234e32ab78327a45741b8a816b4332ee"} +{"family": "injection", "fold": "outer-3", "group": "qwed-ai/qwed-verification", "label": 0, "pair": "", "unit": "fc7df6887ac719d4615934c0e8c3296728c2f6089f233158c59989027c802d72"} +{"family": "path", "fold": "outer-2", "group": "rack/rack", "label": 1, "pair": "c0412509f15fe3e1", "unit": "63bda2a71ad7155ad2f59aa9086efef3fd3710dc82c57f4b32ec8bf561460239"} +{"family": "path", "fold": "outer-2", "group": "rack/rack", "label": 0, "pair": "c0412509f15fe3e1", "unit": "b1c9a070c93764655786c3de5fde536c150e319efe47548614cc544b73a8ee2b"} +{"family": "deserialization", "fold": "outer-3", "group": "rails/kredis", "label": 1, "pair": "feed5f7de11289a4", "unit": "71531567b2b8609ae0d31cb177b8faea4f5fc61923e513b0154a2c941287ce2b"} +{"family": "deserialization", "fold": "outer-3", "group": "rails/kredis", "label": 0, "pair": "feed5f7de11289a4", "unit": "d6be6d397a301a07faa761163d3d9e270d71d23d3bdfd084eeafb1e62b91dd71"} +{"family": "deserialization", "fold": "outer-2", "group": "redisson/redisson", "label": 1, "pair": "ffd4b04a7bebfaf8", "unit": "54f7519d3d59b6d8d69a5f8f7bd78e360d3fabaae00b7587c37e388719169370"} +{"family": "deserialization", "fold": "outer-2", "group": "redisson/redisson", "label": 0, "pair": "ffd4b04a7bebfaf8", "unit": "c5e42c9ab2ee54e46700ab0b64e239b567fbc75ccaceaf6cbf20687813663332"} +{"family": "deserialization", "fold": "outer-0", "group": "remix-run/react-router", "label": 1, "pair": "73f2a014c6a699ed", "unit": "44ba9d9d8e75769c1444ebf37a64941a845b4bcae678ec79d64dc84e5cc067a0"} +{"family": "deserialization", "fold": "outer-0", "group": "remix-run/react-router", "label": 0, "pair": "73f2a014c6a699ed", "unit": "b430d488c486cef26b1c661f773e7e9e4a177eeea73537a033eec98d532e4eed"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "remram44/taguette", "label": 1, "pair": "cb0c638282db0d37", "unit": "a49e6ad2e7c2a5ad0d68d5d69c263306b501fc0dc9382e701fb5fe680611196b"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "remram44/taguette", "label": 0, "pair": "cb0c638282db0d37", "unit": "dee28099d3b37cbd0ee14412122ecc5f8488ed6bec01863520b87706a040fba1"} +{"family": "injection", "fold": "outer-4", "group": "roest01/node-pdf-image", "label": 0, "pair": "09bce271e8530fbc", "unit": "35eb19b4c6176b37287e2123bb98ec4045ce6bde7cc3864dde317c024b0935d8"} +{"family": "injection", "fold": "outer-4", "group": "roest01/node-pdf-image", "label": 1, "pair": "09bce271e8530fbc", "unit": "dd33291cc494bbc77d44e9b13d16f171d32c9746637506f315f375e8a9ddee95"} +{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 1, "pair": "40d8d81a764c508f", "unit": "7deeed73c29c74ccbed29be049e109d363d41e4b5eea245ee57780b85f0e137a"} +{"family": "injection", "fold": "outer-2", "group": "rrainn/portprocesses", "label": 0, "pair": "40d8d81a764c508f", "unit": "b5cdadf848e67910550666ba2a6ac67b24112e501c02dba1865766a316317a23"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "", "unit": "15dbea384b3255457a44d9382efc6944dd27f1d411d903344a5d5e5fd194b72f"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "b2ac284bd70dd684", "unit": "225890ba6694dc8f4fbab46e9c851dff8c095e906cf14db15d01ebce0891aaab"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "b6eca35de93a5be8", "unit": "42eb8427415704e060e73b5ce0943b014ddf380ca756d5ec47c558e59a79895a"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "b6eca35de93a5be8", "unit": "4c9cdcc11a18641ef41e9308712c5e8b830ec9ad5ebc3a956c8701e4f77b39c0"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "6a53939838afbdd7", "unit": "5794809e595311c86909d673a570d003aa025e4dcd6abc5641b641d935beb0b1"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "46d3ec7d49cb8ca7", "unit": "5f7a63ec650915962ee5255f5416df63ab882d6107f3b540be36d481798a8944"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "6a53939838afbdd7", "unit": "97a7018535ab511622e3ade9e53d570ffb1c80633eaa372fb76e749b3a586c8f"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "46d3ec7d49cb8ca7", "unit": "a84e9ab47f4f2b584684dbb81575758d9032fe3f2ed53bef478cb9dcfedaf677"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "b2ac284bd70dd684", "unit": "aeceb1ebc863ebbc3b69ae56c5b78d2dba005132b75cbd6af929bb0bfe25da9b"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 1, "pair": "f42cab375ace1739", "unit": "e329d4a6ac757680bf3d292a81f1f74f8a8a9b482d1651efe9f503f6a4de8796"} +{"family": "injection", "fold": "outer-3", "group": "saltcorn/saltcorn", "label": 0, "pair": "f42cab375ace1739", "unit": "e73c5aac41c0b93072223a3097fb175548ce893e0678ff3a22bc7571745a17e0"} +{"family": "injection", "fold": "outer-2", "group": "sgabe/dsvpwa", "label": 1, "pair": "", "unit": "52354ccdbc16f209e4692d97d5237e04ed030cf4a5e71f77b5a767962b355b4a"} +{"family": "injection", "fold": "outer-2", "group": "sgabe/dsvpwa", "label": 1, "pair": "7d23be667536afff", "unit": "b5016a5a53409491ba805ed8718599b84250b7b6914da0434b0882b0b3578dc8"} +{"family": "injection", "fold": "outer-2", "group": "sgabe/dsvpwa", "label": 0, "pair": "7d23be667536afff", "unit": "f1b01102968568d3ba1b2fdc70f4ae281f433fb20d29eff2a55fc45555ddbfcd"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 0, "pair": "1199de75f426940b", "unit": "10fdc236f43a02b7e291eb8bf6ada297a2ae98086a63b171057bc7bfc37a3377"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 0, "pair": "22112c31603bbe9f", "unit": "5eb3b7523b5a196dcaacba08d68457f9a45e31503ea02745ca3a7d0542f9881c"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 1, "pair": "22112c31603bbe9f", "unit": "6bd5bdfd81b0a3374d59cfb2c1f7fa2b27887f4672f4485e77eb5234b569983f"} +{"family": "untrusted_destination", "fold": "outer-4", "group": "signalk/signalk-server", "label": 1, "pair": "1199de75f426940b", "unit": "c09dffd04708540845c8d1dd77a2bd989bd35f264abf652a2db55f8cb1faf7a9"} +{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 0, "pair": "7aab025ea7b97019", "unit": "7487a34cdfdb532a5096764a47a3e565fa509605c1df0e2b105eb9c751fc5307"} +{"family": "output_encoding", "fold": "outer-1", "group": "silverstripe/silverstripe-versioned", "label": 1, "pair": "7aab025ea7b97019", "unit": "9c0be2a58741f095fd0a9cdc60d4ca4039de1529699d5701dc4d0bcf7d5bf90c"} +{"family": "path", "fold": "outer-0", "group": "simonh1000/angular-http-server", "label": 0, "pair": "4de5441f1776f083", "unit": "a3a35d573da308cd1d0614a26616bbd37b360d9ea416894cdeffdb59abd691d4"} +{"family": "path", "fold": "outer-0", "group": "simonh1000/angular-http-server", "label": 1, "pair": "4de5441f1776f083", "unit": "dc33fc26c1bf7c67e5439ddfdad06c0faf16c796e0fe8a46b6e6a541e708676d"} +{"family": "access_control", "fold": "outer-3", "group": "smithyhq/sqladmin", "label": 1, "pair": "d76d5232a8587127", "unit": "1ec32d427822d1ddd3b0d7245f5771ddeb56c146ac8ff9959dd85760ff659a20"} +{"family": "access_control", "fold": "outer-3", "group": "smithyhq/sqladmin", "label": 0, "pair": "d76d5232a8587127", "unit": "5143b19a1a1e6b9162b80594ed7fb5c0c57ed0b8abfd793bfc71089552d57638"} +{"family": "injection", "fold": "outer-1", "group": "soyuka/pidusage", "label": 0, "pair": "5f6e814444b4acd8", "unit": "2e8aa07b1aa4bfc5349b0d4464f3f2a6500aa55ff765b0139e9901ed34151ffa"} +{"family": "injection", "fold": "outer-1", "group": "soyuka/pidusage", "label": 1, "pair": "5f6e814444b4acd8", "unit": "e26c726c08be761b22fcc6ec17fc1fa7aba4769f08957f024e5545906df62797"} +{"family": "output_encoding", "fold": "outer-2", "group": "spatie/schema-org", "label": 1, "pair": "888e955fb6cfb4fd", "unit": "6f1a431ca97afc34ceec48393eb134a8928a312e6f761eb907c6b5855cac6208"} +{"family": "output_encoding", "fold": "outer-2", "group": "spatie/schema-org", "label": 0, "pair": "888e955fb6cfb4fd", "unit": "7b62415d17569924d16238a4aa28ab1af2db6561f56be9f59d8075fd1bc4a941"} +{"family": "injection", "fold": "outer-3", "group": "strangerstudios/paid-memberships-pro", "label": 1, "pair": "", "unit": "ac905d12b92954263c046bd0138a682cea86cf8476336d6baa664ade28d34ac6"} +{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 1, "pair": "218a45d869bd7e39", "unit": "309ebac468b7ab054c8396ed495770385ae3590caba43709745ba2d1db4b6d54"} +{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 1, "pair": "", "unit": "3cc8613813dec13cb1b7b45b296688da883f0b36cd2c4fc1da6e81703fb59ab9"} +{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 0, "pair": "218a45d869bd7e39", "unit": "9bd2331dbc260451355307e4c37bae17ad6676654c66181934e1b35fb174cc5a"} +{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "b087ac4d0989e0b72698a9644ef4eac8073b4a15bb21730d63e4730b5cc58a60"} +{"family": "injection", "fold": "outer-3", "group": "sulu/sulu", "label": 0, "pair": "", "unit": "c5d1e8d19560c997018fdca12eb60f1799be76c33be058b88472f5a2fc6709be"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "22befc17a75a688e2f9358d2ddf9a7b197da44cd95dbdeae5e0191b113ee54d3"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 0, "pair": "63f496d9ff53e39b", "unit": "6a98f8145776c78793b37156e9a81757841bb6de80016ab4fa575209d724db3f"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 1, "pair": "63f496d9ff53e39b", "unit": "7ac60c96c15f8517e089cc2240aedcf1fc30e54a41e35fe03c6b965bc393fbd6"} +{"family": "untrusted_destination", "fold": "outer-3", "group": "themegrill/user-registration", "label": 0, "pair": "", "unit": "ae39cb55ef2a87c5347e0ddc40975f142450e82de4909e93ba62a7ed0efb4134"} +{"family": "injection", "fold": "outer-4", "group": "thenables/thenify", "label": 1, "pair": "d1b19898f8deaa41", "unit": "3763c7b407ad471820b5f8fa25bde0ba6d91e15c07c56681de4b41add418b0b1"} +{"family": "injection", "fold": "outer-4", "group": "thenables/thenify", "label": 0, "pair": "d1b19898f8deaa41", "unit": "bc8b934938c30687f3d4f982392d2cfa9504a916b463ccb5c46fe8a30eede488"} +{"family": "path", "fold": "outer-1", "group": "thumbor/thumbor", "label": 1, "pair": "f5f33fa6fe7b8492", "unit": "938bbf146a9037e687190bcee82c047492dffbe90efea40a49a647f81a0f1ed0"} +{"family": "path", "fold": "outer-1", "group": "thumbor/thumbor", "label": 0, "pair": "f5f33fa6fe7b8492", "unit": "b90c9f7e0011685f1a8e853a9ed0ae37fd437ef6d6eb0ba17c240aec2c22b603"} +{"family": "deserialization", "fold": "outer-3", "group": "tomasnorre/crawler", "label": 0, "pair": "f74364dc8e35c8ba", "unit": "11e760e99c5c852676c9288e18a250febb3854e9047b51e8b9ecc9f3b423b6f5"} +{"family": "deserialization", "fold": "outer-3", "group": "tomasnorre/crawler", "label": 1, "pair": "f74364dc8e35c8ba", "unit": "5fc980194040f4c4d803075d4085618cc0260ae8f3454b26b9b704d30200ffea"} +{"family": "output_encoding", "fold": "outer-3", "group": "tornadoweb/tornado", "label": 0, "pair": "db1c51833994ec55", "unit": "c2ce8320c778dd55434d7413844797b68e6fcabc4dc509458a9d60ca70b88c60"} +{"family": "output_encoding", "fold": "outer-3", "group": "tornadoweb/tornado", "label": 1, "pair": "db1c51833994ec55", "unit": "f2d6843547af1276ae8a4dff3a7786913fa79dc69540d4fbb95c4c091e27c7f2"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "truelayer/truelayer-dotnet", "label": 1, "pair": "f4690679bf9d80aa", "unit": "1a4e24df8d08672ecde4709a77ea5c437b2198fecff8bfff373b63c9eaed6e63"} +{"family": "untrusted_destination", "fold": "outer-1", "group": "truelayer/truelayer-dotnet", "label": 0, "pair": "f4690679bf9d80aa", "unit": "78711973ec84867671314dd7878769092b32f59860eafed50a0f6979daee76fc"} +{"family": "injection", "fold": "outer-2", "group": "ultimatemember/ultimatemember", "label": 0, "pair": "85c15e3fc2b370b2", "unit": "0990d8f41cb07c1cf217812e51a7025381de4ba68d1be3799e26241d81dcea67"} +{"family": "injection", "fold": "outer-2", "group": "ultimatemember/ultimatemember", "label": 1, "pair": "85c15e3fc2b370b2", "unit": "9da1d63f0c4da270f66eb87bf9c342ace883a0aef78c4643e254159b5da8fce3"} +{"family": "output_encoding", "fold": "outer-0", "group": "viewcomponent/view_component", "label": 0, "pair": "8f5a6abedd742bbe", "unit": "71e3ced643a244100cc3fde234f5b4b27655214c7ae0ff67ab61e3e48a8fdf78"} +{"family": "output_encoding", "fold": "outer-0", "group": "viewcomponent/view_component", "label": 1, "pair": "8f5a6abedd742bbe", "unit": "7c8cbfae87127042f9f43621345b4f7958bde27a3ff9db76254063e80735f3ce"} +{"family": "injection", "fold": "outer-1", "group": "vincentmorneau/apex-publish-static-files", "label": 1, "pair": "c9538342f2d7efca", "unit": "2b8db57d65ef1d67c457ff5a336e8a542d03387395d69d5e16815c4f00f40723"} +{"family": "injection", "fold": "outer-1", "group": "vincentmorneau/apex-publish-static-files", "label": 0, "pair": "c9538342f2d7efca", "unit": "e2ed113dc21ad5f7caa71ba4a5ece8f8f900a97fd0d00dc85f7ac79ff5bed19a"} +{"family": "injection", "fold": "outer-1", "group": "vvo/node-whereis", "label": 1, "pair": "8acd426c5771b37c", "unit": "983e146a60e061254b15dc8dd573cbba57f8dd0358f23ea1b9c313ef69069af5"} +{"family": "injection", "fold": "outer-1", "group": "vvo/node-whereis", "label": 0, "pair": "8acd426c5771b37c", "unit": "f80dfec45231e4778078cb10f00128ed445177cac0a062fc2df5ced5c0171f21"} +{"family": "deserialization", "fold": "outer-4", "group": "waterfutures/epyt-flow", "label": 0, "pair": "9b84800ce5c395cb", "unit": "9b7dfdceae03fd86ad3260a03f85bcdcff8f32f10c265d23743ff8dc98eeefb8"} +{"family": "deserialization", "fold": "outer-4", "group": "waterfutures/epyt-flow", "label": 1, "pair": "9b84800ce5c395cb", "unit": "b99dda150c0da3defa5f4c527ef6c402f5589dfb2f2f03f3e5e424910eb48c53"} +{"family": "path", "fold": "outer-4", "group": "webpack/webpack-dev-middleware", "label": 1, "pair": "ad64e8fac4ac3ee0", "unit": "36dfd01915a553d86576e8ecd3b737b8e2b297aa542f58cc83b448060e74398a"} +{"family": "path", "fold": "outer-4", "group": "webpack/webpack-dev-middleware", "label": 0, "pair": "ad64e8fac4ac3ee0", "unit": "c2b6d40f130d35bbfc16fab9411c664fa06321c59fb1d901a1987a69cb09c72e"} +{"family": "access_control", "fold": "outer-4", "group": "wintercms/winter", "label": 0, "pair": "d48c08b19f560848", "unit": "03d02c9e25eec0f380b125279fce35e8867065bf6e1ec75f9f39729ce6efdc77"} +{"family": "access_control", "fold": "outer-4", "group": "wintercms/winter", "label": 1, "pair": "d48c08b19f560848", "unit": "abbea69b7d0fa68c3ace442c25597e128cde354b417787b5913fd10d666c0d19"} +{"family": "output_encoding", "fold": "outer-3", "group": "xyproto/algernon", "label": 0, "pair": "3844ed3d703d7ef4", "unit": "445efda074e4d9f6c1634225eb08a3c3eb1257311c995ce891df2c791a6f5c8b"} +{"family": "output_encoding", "fold": "outer-3", "group": "xyproto/algernon", "label": 1, "pair": "3844ed3d703d7ef4", "unit": "8ac27b08549353ada819caaf99a5180d2f2f515b03aa8cd99a5d37b527ea3a5c"} +{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 1, "pair": "a63eae7d621a985e", "unit": "04ca0b583b09c3b4f67bd2496a4607ecb134d24b7bdaf8960fa151e40ebcc447"} +{"family": "path", "fold": "outer-3", "group": "yiisoft/yii2", "label": 0, "pair": "a63eae7d621a985e", "unit": "ecfa2a67726ca9d3e2621a4d8be60145095b76aa18ecf0268212341d63752871"} +{"family": "deserialization", "fold": "outer-4", "group": "zumba/json-serializer", "label": 0, "pair": "0d9f529b7f0084b6", "unit": "35aa369236f510634b3aa7b54b04e9da0f80877e515786a888e760f8fe33c524"} +{"family": "deserialization", "fold": "outer-4", "group": "zumba/json-serializer", "label": 1, "pair": "0d9f529b7f0084b6", "unit": "ef4f8613c0914eb815504e00cd6ee05df38352fd3b27826798e6623594017042"} diff --git a/plane/experiments/exp-003-precision-bound-blocking.yaml b/plane/experiments/exp-003-precision-bound-blocking.yaml new file mode 100644 index 0000000..05ae5c0 --- /dev/null +++ b/plane/experiments/exp-003-precision-bound-blocking.yaml @@ -0,0 +1,65 @@ +# Experiment exp-003 (learned-decision-engine Req 6, Req 7.3/7.4): the BLOCK rule. Registered before it runs; +# `ousast learn experiment register` records this file's sha256, and a changed file is a new experiment. +# kind: rule -- both arms decide from the SAME scores: one scoring pass of the six slice-2 incumbent programs, replayed +# from the response cache only (zero cost by construction: a zero-ceiling meter around a refusing client; a unit with any +# uncached response is excluded and counted, never asked; the meter's calls and usd must read 0 or the run is void). +# +# Threshold selection (arm B), fixed here: per family and per outer fold j, the candidate thresholds are the raw scores +# s of the training rows, i.e. the family's paired units in the OTHER four outer folds (each of those predictions is +# itself out of repository: retrieval and demonstrations never cross its own fold's boundary). The chosen threshold is +# the lowest t such that, among those training rows with s >= t whose majority verdict is not `unsure`, the Wilson 95% +# lower bound of precision is >= 0.95. No such t -> fold j gets no BLOCK. Fold j's units are then flagged when not +# `unsure` and s >= t. Fold j's labels never enter its own threshold (tests/test_learn_rules.py). +# +# Pre-registered expectation (power): a Wilson lower bound of 0.95 needs at least 73 flagged units with no false flag. +# The frozen units (353 in 146 repository groups, 244 paired) hold 38 paired positives for injection and 16-19 for each +# other family, all five folds together; the training rows of any one fold hold fewer. So arm B cannot offer a BLOCK in +# any family by construction, and the expected verdict is inconclusive with arm B flagging nothing. That is the finding +# this run would confirm: on this data the bound, not the calibration gate, is what makes BLOCK unreachable. Arm C (point precision >= 0.95 over >= 10 training flags) is a secondary, +# non-deciding arm that measures how far the held-out precision is from the bar when the bound is relaxed. +id: exp-003-precision-bound-blocking +hypothesis: > + The calibrated-probability BLOCK rule (Platt map + calibration gate, ECE <= 0.05) is the wrong instrument for M4, + which asks for precision on what is flagged. Selective blocking -- a per-family raw-score threshold chosen on training + folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95 -- reaches the M4 precision + bar on held-out repositories where the calibrated rule offers no BLOCK. +kind: rule +programs: # the slice-2 incumbents (benchmarks/measurements/2026-10-02-decision-engine-slice-2, program.id) + access_control: 745cb1d76bbb09dc1574d333d2118489be62e39886517894d3699fa565b0e8c2 + deserialization: 3c0c2831be7d78198a62351661c37867363c501cb40e3ed7b0119c155af09654 + injection: b207a0710e0abf053b3b9397b25f9ee1cab4679144865e3a82c1b9be2c7b4982 + output_encoding: 5d768f16724562c381c09949ff3df9cbb11d58cbb7a9c0515d0dc3f858d3818e + path: aea396680a4874dcb6de70e240ca0480c83424debdd3710bc3f1ea6a612661f3 + untrusted_destination: 1242b3973fc89056aa08e00843043d68235df217d8e3ef8a0ea0e0b81c142753 +arms: + A: + label: today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p + rule: {name: calibrated_block} + B: + label: selective blocking -- per-family raw-score threshold from the other folds' paired units, lowest t with Wilson 95% LB of flagged precision >= 0.95 + rule: {name: precision_bound_block, bound: wilson, precision: 0.95, select_on: paired} + C: + label: secondary, non-deciding -- as B with point precision >= 0.95 over at least 10 training flags + rule: {name: precision_bound_block, bound: point, precision: 0.95, min_flagged: 10, select_on: paired} +unit: candidate +pairing: paired # every unit scored once; every arm decides every unit +cluster: repository +order: {randomise: none, seed: 3} +# primary population: paired units (both sides of the candidate present). It excludes every positives-only group and any +# unpaired unit, whose positives never had to be told apart from their own fix. Pooled (every unit) is reported beside it. +populations: {primary: paired, secondary: [pooled]} +metric: + primary: + - {name: flagged_precision_lb, target: 0.95} # held-out (outer-fold) Wilson 95% LB of flagged precision, per family, arm B, paired + secondary: [flagged_precision, coverage, flagged] # coverage = flagged true positives / positives +# Decision rule (decide_rule): REJECT B if in any family B flags held-out paired units at a point precision < 0.95; +# else ADOPT B if in some family B's held-out paired Wilson LB >= 0.95; else INCONCLUSIVE (the incumbent rule stays). +# A family "reaches the M4 precision bar on held-out repositories" when its held-out paired Wilson LB >= 0.95 (the +# point precision is reported beside it). +decision: {arm: B, target_precision: 0.95} +test: {resamples: 2000} # repository-cluster bootstrap of the held-out flagged precision, beside the Wilson bound +mde: 0.0 # not a paired difference test; see the power note above +budget_usd: {total: 0} # zero cost is a hard rule +families: [access_control, deserialization, injection, output_encoding, path, untrusted_destination] +folds: {kind: outer, k: 5, seed: 0, compile_fraction: 0.25} # the incumbent evaluation's folds (learn/compile.toml seed) +units_file: plane/experiments/exp-003-precision-bound-blocking.units.jsonl # frozen by `ousast learn experiment units` diff --git a/src/openultrasast/learn/cli.py b/src/openultrasast/learn/cli.py index a2f1eb2..fe3e4aa 100644 --- a/src/openultrasast/learn/cli.py +++ b/src/openultrasast/learn/cli.py @@ -122,11 +122,17 @@ def _experiment(args: argparse.Namespace) -> int: print(json.dumps({k: row[k] for k in shown}, indent=2)) return 0 if args.experiment_command == "analyse": - row = ex.check_registered(store, manifest, need_units=False) + row = ex.check_registered(store, manifest, need_units=manifest.kind == "rule") provenance = {"code_commit": ex.committed_at_head(manifest.path)} - report = ex.analyse(manifest, ex.load_outcomes(store, manifest.id), resamples=args.resamples, provenance=provenance) + outcomes = ex.load_outcomes(store, manifest.id) + if manifest.kind == "rule": + units = ex.read_units(manifest.units_file) + report = ex.analyse_rule(manifest, outcomes, units, resamples=args.resamples, provenance=provenance) + else: + report = ex.analyse(manifest, outcomes, resamples=args.resamples, provenance=provenance) if args.record: - store.put_row(ex.result_row(manifest, str(row["pin"]), report)) + result = ex.rule_result_row if manifest.kind == "rule" else ex.result_row + store.put_row(result(manifest, str(row["pin"]), report)) _write(report, args.out or (ex.RESULT_DIR / manifest.id / "result.json" if args.record else None)) return 0 from .embeddings import EmbeddingCache @@ -143,6 +149,16 @@ def _experiment(args: argparse.Namespace) -> int: cache = EmbeddingCache(store) vectors = {sha: v for sha in cache.names() if (v := cache.get(sha)) is not None} or None parameters = _model_parameters(args.manifest_model) + + def excerpt_text(sha: str) -> str | None: + data = store.get_blob("excerpts", sha) + return data.decode("utf-8") if data is not None else None + + if manifest.kind == "rule": # zero cost: cache only, behind a zero-ceiling meter; no endpoint is resolved + caller, meter = ex.zero_cost_caller(_arm_model(manifest, manifest.arms["A"], store), parameters, store) + summary = ex.run_rule(store, manifest, memory, caller, meter, excerpt_text, vectors) + _write(summary.as_dict(), args.out) + return 0 callers: dict[str, Caller] = {} for name, arm in manifest.arms.items(): if arm.replay_only: @@ -157,10 +173,6 @@ def _experiment(args: argparse.Namespace) -> int: client = MeteredClient(resolved[0], prices=parameters, budget_usd=arm.budget_usd) callers[name] = Caller(client, _arm_model(manifest, arm, store), parameters, store=store) - def excerpt_text(sha: str) -> str | None: - data = store.get_blob("excerpts", sha) - return data.decode("utf-8") if data is not None else None - early = not args.no_early_stop summary = ex.run(store, manifest, memory, callers, excerpt_text, vectors, stop_early=early, resamples=args.resamples) except ex.ExperimentError as exc: diff --git a/src/openultrasast/learn/evaluate.py b/src/openultrasast/learn/evaluate.py index e15e7dc..12a6621 100644 --- a/src/openultrasast/learn/evaluate.py +++ b/src/openultrasast/learn/evaluate.py @@ -54,6 +54,7 @@ class Prediction: parse_failed: int = 0 p: float | None = None point: str | None = None # BLOCK | ADVISORY | DROP, from points chosen on the other folds + pair: str = "" # the two sides of one candidate share it; "" when only one side is present @property def blockable(self) -> bool: @@ -128,6 +129,87 @@ def nested_points(preds: Sequence[Prediction]) -> tuple[list[Prediction], dict[s return out, points +BOUNDS = ("wilson", "point") + + +def precision_threshold( + rows: Sequence[Prediction], *, precision: float = TARGET_PRECISION, bound: str = "wilson", min_flagged: int = 1 +) -> float | None: + """The lowest raw-score threshold ``t`` at which the blockable rows with ``s >= t`` reach ``precision``: their + Wilson 95% lower bound (``bound="wilson"``) or their point precision over at least ``min_flagged`` rows + (``bound="point"``). Candidate thresholds are the rows' own scores; ``None`` when no threshold qualifies.""" + if bound not in BOUNDS: + raise ValueError(f"bound {bound!r} (one of {', '.join(BOUNDS)})") + blockable = [r for r in rows if r.blockable] + for t in sorted({r.s for r in blockable}): + chosen = [r for r in blockable if r.s >= t] + hits = sum(r.label for r in chosen) + if len(chosen) < max(1, min_flagged): + continue + level = wilson_lower(hits, len(chosen)) if bound == "wilson" else hits / len(chosen) + if level >= precision: + return t + return None + + +def nested_precision_block( + preds: Sequence[Prediction], + *, + eligible: Callable[[Prediction], bool] | None = None, + precision: float = TARGET_PRECISION, + bound: str = "wilson", + min_flagged: int = 1, +) -> tuple[list[Prediction], dict[str, dict[str, float | None]]]: + """Selective blocking on the raw score, per family: the threshold for outer fold j is chosen on the *other* folds' + rows of the family only (and, with ``eligible``, only those rows); the held-out fold's labels never enter it. + Every row of fold j is then decided with that threshold: ``BLOCK`` when blockable and ``s >= t``, else ``DROP``. + A family whose training folds reach no threshold offers no BLOCK on that fold. Returns the decided rows and the + thresholds by family and fold.""" + thresholds: dict[str, dict[str, float | None]] = {} + for family in sorted({r.family for r in preds}): + rows = [r for r in preds if r.family == family] + thresholds[family] = { + fold: precision_threshold( + [r for r in rows if r.fold != fold and (eligible is None or eligible(r))], + precision=precision, bound=bound, min_flagged=min_flagged, + ) + for fold in sorted({r.fold for r in rows}) + } # fmt: skip + out = [] + for r in preds: + t = thresholds[r.family][r.fold] + out.append(replace(r, point="BLOCK" if t is not None and r.blockable and r.s >= t else "DROP")) + return out, thresholds + + +def calibrated_block(preds: Sequence[Prediction], *, seed: int = 0, **floors: int) -> tuple[list[Prediction], dict[str, Any]]: + """Today's BLOCK rule, per family as the slice-2 evaluation applied it: the Platt map cross-fitted on the other + folds, the calibration check on the family's held-out calibrated predictions, the BLOCK point chosen on the other + folds' calibrated predictions (nested); a row is ``BLOCK`` only when the family's calibration holds. Returns the + decided rows and, per family, the check and the calibrated thresholds by fold.""" + out: list[Prediction] = [] + info: dict[str, Any] = {} + for family in sorted({r.family for r in preds}): + rows = [r for r in preds if r.family == family] + ps, _ = cross_fit([Row(r.fold, r.family, r.group, r.label, r.s) for r in rows], **floors) + calibrated = [replace(r, p=p) for r, p in zip(rows, ps, strict=True)] + decided = [r for r in calibrated if r.p is not None] + checked = ( + check([float(r.p) for r in decided if r.p is not None], [r.label for r in decided], [r.group for r in decided], seed=seed) + if len({r.label for r in decided}) == 2 + else None + ) + holds = bool(checked and checked.holds) + nested, points = nested_points(calibrated) + out += [replace(r, point="BLOCK" if holds and r.point == "BLOCK" else "DROP") for r in nested] + info[family] = { + "calibration_holds": holds, "ece": None if checked is None else round(checked.ece, 4), + "thresholds_by_fold": {f: p.block for f, p in points.items()}, + "reason_by_fold": {f: ("calibration" if not holds else p.block_reason) for f, p in points.items()}, + } # fmt: skip + return out, info + + def _at(rows: Sequence[Prediction], level: str) -> dict[str, float | None]: flagged = [r for r in rows if r.point in (("BLOCK",) if level == "block" else ("BLOCK", "ADVISORY"))] positives = sum(r.label for r in rows) @@ -326,8 +408,9 @@ def candidate_from_example( __all__ = [ - "PARSE_FAILURE_LIMIT", "TARGET_PRECISION", "TARGET_RECALL", "Points", "Prediction", "candidate_from_example", "choose_points", - "cluster_bootstrap", "evaluate", "explain", "floor_status", "incompatible", "learning_curve", "nested_points", "predict", "strata", - "summary", "targets", "unevaluable_families", + "BOUNDS", "PARSE_FAILURE_LIMIT", "TARGET_PRECISION", "TARGET_RECALL", "Points", "Prediction", "calibrated_block", + "candidate_from_example", "choose_points", "cluster_bootstrap", "evaluate", "explain", "floor_status", "incompatible", + "learning_curve", "nested_points", "nested_precision_block", "precision_threshold", "predict", "strata", "summary", "targets", + "unevaluable_families", "wilson_lower", ] # fmt: skip diff --git a/src/openultrasast/learn/experiments.py b/src/openultrasast/learn/experiments.py index ee54b04..a62ca0e 100644 --- a/src/openultrasast/learn/experiments.py +++ b/src/openultrasast/learn/experiments.py @@ -41,10 +41,10 @@ from typing import Any import yaml -from ..plane.budget import BudgetExhausted +from ..plane.budget import BudgetExhausted, MeteredClient from ..plane.memory import MemoryStore, row_id from .compile import load_program, spec_of -from .evaluate import candidate_from_example +from .evaluate import BOUNDS, TARGET_PRECISION, Prediction, calibrated_block, candidate_from_example, nested_precision_block, wilson_lower from .examples import EXAMPLE_KIND, Example from .folds import Fold, compile_split, folds_digest, outer_folds from .program import Caller, Decision, Program, ProgramSpec @@ -57,7 +57,15 @@ RESULT_KIND = "experiment_result" REGISTER_TASK = "register" POPULATION = "experiments" # the envelope of every experiment row; `split` is the experiment id RUN_TASK = "run" -ARM_KINDS = ("program", "plane") +ARM_KINDS = ("program", "plane", "rule") +# decision-rule experiments (kind: rule): one replay-only scoring pass of the incumbent programs, arms differ only in +# the BLOCK rule applied to the same scores +RULES = ("calibrated_block", "precision_bound_block") +POPULATIONS = ("paired", "pooled") # paired: units whose candidate has both sides present; pooled: every unit +SCORE_ARM = "score" # the arm name of a rule experiment's outcome rows (the shared scores) +MISS_VERDICT = "replay_miss" # an outcome row of a unit excluded because a response was not cached +# rule metrics, per family and population of held-out (outer-fold) units: direction only (no paired test) +RULE_METRICS: Mapping[str, int] = {"flagged_precision_lb": 1, "flagged_precision": 1, "coverage": 1, "flagged": 1} UNITS = ("candidate", "file", "repository") PAIRINGS = ("paired", "assigned") # O'Brien-Fleming boundaries for two equally spaced looks at alpha 0.05, two-sided (design section 6). @@ -96,6 +104,7 @@ class Arm: replay_only: bool = False repeats: int = 1 budget_usd: float = 0.0 + rule: Mapping[str, Any] = field(default_factory=dict) # rule arms: {name: calibrated_block | precision_bound_block, ...} @dataclass(frozen=True) @@ -119,11 +128,11 @@ class Metric: @property def direction(self) -> int: - return METRICS[self.name][0] + return RULE_METRICS[self.name] if self.name in RULE_METRICS else METRICS[self.name][0] @property def binary(self) -> bool: - return METRICS[self.name][1] + return self.name in METRICS and METRICS[self.name][1] def as_dict(self) -> dict[str, Any]: return {k: v for k, v in asdict(self).items() if v is not None} @@ -198,6 +207,7 @@ def load_manifest(path: Path) -> Manifest: bool(spec.get("replay_only", False)), int(spec.get("repeats", 1)), float(spec.get("budget_usd", 0.0)), + dict(spec.get("rule") or {}), ) # fmt: skip for name, spec in dict(raw["arms"]).items() } @@ -210,9 +220,12 @@ def load_manifest(path: Path) -> Manifest: primary_raw = metric["primary"] primary = tuple(Metric.parse(m) for m in (primary_raw if isinstance(primary_raw, list) else [primary_raw])) secondary = tuple(Metric.parse(m) for m in metric.get("secondary", [])) + known = RULE_METRICS if kind == "rule" else METRICS for m in (*primary, *secondary): - if m.name not in METRICS: - raise ExperimentError(f"{path}: metric {m.name!r} (one of {', '.join(METRICS)})") + if m.name not in known: + raise ExperimentError(f"{path}: metric {m.name!r} (one of {', '.join(known)})") + if kind == "rule": + _check_rule_manifest(path, raw, arms) test = dict(raw.get("test") or {}) stopping = dict(raw.get("stopping") or {"looks": [1.0], "boundaries": [1.959964]}) looks = tuple(float(x) for x in stopping.get("looks", [0.5, 1.0])) @@ -244,11 +257,39 @@ def load_manifest(path: Path) -> Manifest: raise ExperimentError(f"{path}: missing {exc}") from exc if manifest.id != path.stem: raise ExperimentError(f"{path}: id {manifest.id!r} must equal the file stem") - if kind == "program" and not manifest.families: - raise ExperimentError(f"{path}: a program experiment names its families") + if kind in ("program", "rule") and not manifest.families: + raise ExperimentError(f"{path}: a {kind} experiment names its families") + if kind == "rule" and set(manifest.families) - set(manifest.programs): + raise ExperimentError(f"{path}: a rule experiment names a program for every family (programs:)") return manifest +def _check_rule_manifest(path: Path, raw: Mapping[str, Any], arms: Mapping[str, Arm]) -> None: + """A decision-rule experiment costs nothing by construction: no arm overrides the program, no budget, and every + arm names a known rule with known parameters; the populations are declared before the run.""" + budget = raw["budget_usd"] + total = float(budget["total"] if isinstance(budget, Mapping) else budget) + if total != 0 or any(a.budget_usd for a in arms.values()): + raise ExperimentError(f"{path}: a rule experiment replays cached responses only: budget_usd must be 0") + for arm in arms.values(): + if arm.spec or arm.task_env or arm.programs: + raise ExperimentError(f"{path}: arm {arm.name} of a rule experiment may change the BLOCK rule only (rule:)") + name = arm.rule.get("name") + if name not in RULES: + raise ExperimentError(f"{path}: arm {arm.name} rule.name {name!r} (one of {', '.join(RULES)})") + allowed = {"name"} | ({"precision", "bound", "min_flagged", "select_on"} if name == "precision_bound_block" else set()) + if set(arm.rule) - allowed: + raise ExperimentError(f"{path}: arm {arm.name} rule parameters {sorted(set(arm.rule) - allowed)} are not understood") + if arm.rule.get("bound", "wilson") not in BOUNDS or arm.rule.get("select_on", "paired") not in POPULATIONS: + raise ExperimentError(f"{path}: arm {arm.name} bound in {BOUNDS} and select_on in {POPULATIONS}") + populations = dict(raw.get("populations") or {}) + if populations.get("primary") not in POPULATIONS or any(p not in POPULATIONS for p in populations.get("secondary", [])): + raise ExperimentError(f"{path}: populations.primary (and secondary) in {POPULATIONS}") + decision = dict(raw.get("decision") or {}) + if not 0 < float(decision.get("target_precision", 0)) <= 1 or decision.get("arm") not in arms: + raise ExperimentError(f"{path}: decision.target_precision in (0, 1] and decision.arm naming an arm") + + # --- register ------------------------------------------------------------------------------------------------------------ @@ -383,7 +424,7 @@ def program_folds(memory: Sequence[Example], manifest: Manifest) -> tuple[list[F def freeze_units(store: MemoryStore, manifest: Manifest, memory: Sequence[Example]) -> list[Unit]: """The evaluation candidates of the manifest's families under the incumbent's outer folds, paired by candidate across the two pins where both sides are in memory (no model; the list is then committed and registered).""" - if manifest.kind != "program": + if manifest.kind not in ("program", "rule"): raise ExperimentError(f"{manifest.id}: units of a plane experiment come from candidate generation, not the memory") folds, _ = program_folds(memory, manifest) sides: dict[str, tuple[str, str, str]] = {} @@ -812,6 +853,8 @@ def run( """Every unit in every arm, paired, repositories in the pre-shuffled look order, a seeded coin per unit for which arm goes first; outcomes already in the store are not decided again. After the first look's repositories the primary is tested against its boundary and, when crossed, the run stops. A ceiling ends the run ``unfinished``.""" + if manifest.kind == "rule": + raise ExperimentError(f"{manifest.id}: a rule experiment scores once at zero cost (run_rule), its arms differ only in the rule") if manifest.kind != "program": raise ExperimentError(f"{manifest.id}: plane arms run as plane Runs; their outcomes arrive as arm_outcome rows") row = check_registered(store, manifest) @@ -892,7 +935,267 @@ def run( return summary +# --- decision-rule experiments (kind: rule) ----------------------------------------------------------------------------- + + +class ZeroCostViolation(ExperimentError): + """A model call reached the client of a zero-cost run, or the meter recorded spend: the run is void.""" + + +class _RefusingChat: + """The inner client behind a zero-cost run's meter. The meter's zero ceilings stop every call before it gets here; + reaching it means a guard was bypassed.""" + + usage: list[Any] = [] + + def complete(self, **_: Any) -> Any: + raise ZeroCostViolation("a model call reached the client of a zero-cost run") + + +def zero_cost_caller(model: str, prices: Mapping[str, Any], store: MemoryStore | None) -> tuple[Caller, MeteredClient]: + """A caller that answers only from the response cache: its client is a meter with zero ceilings (USD and calls) + around a client that refuses. A response that is not cached raises ``BudgetExhausted`` before any call; the run + excludes that unit and counts it. The meter is the proof: its ``calls`` and ``usd`` must stay 0.""" + meter = MeteredClient(_RefusingChat(), prices=prices, budget_usd=0.0, budget_calls=0) # type: ignore[arg-type] + return Caller(meter, model, prices, store=store), meter + + +def _checked_units(manifest: Manifest, memory: Sequence[Example]) -> tuple[list[Unit], dict[str, Fold], str]: + units = read_units(manifest.units_file) + folds, fdigest = program_folds(memory, manifest) + by_name = {f.name: f for f in folds} + index = {e.id for e in memory} + for unit in units: + fold = by_name.get(unit.fold) + if fold is None or unit.group not in fold.eval_groups or unit.unit not in index: + raise ExperimentError( + f"{manifest.id}: unit {unit.unit[:12]} is not under fold {unit.fold!r} of this memory; the memory changed" + ) + return units, by_name, fdigest + + +def run_rule( + store: MemoryStore, + manifest: Manifest, + memory: Sequence[Example], + caller: Caller, + meter: MeteredClient, + excerpt_text: Callable[[str], str | None], + vectors: Mapping[str, Sequence[float]] | None = None, + *, + on_decision: Callable[[Outcome, Decision], None] | None = None, +) -> RunSummary: + """The one scoring pass of a rule experiment: every unit decided once by its family's incumbent program, answered + only from the response cache. A unit whose responses are not all cached is excluded and counted (an outcome row + with ``verdict = replay_miss`` and no score), never asked. Raises :class:`ZeroCostViolation` when the meter shows + a call or any spend afterwards.""" + if manifest.kind != "rule": + raise ExperimentError(f"{manifest.id}: run_rule runs decision-rule experiments only") + if caller.client is not meter or meter.budget_usd != 0.0 or meter.budget_calls != 0: + raise ZeroCostViolation(f"{manifest.id}: a rule experiment runs only behind a zero-ceiling meter (zero_cost_caller)") + row = check_registered(store, manifest) + units, by_name, fdigest = _checked_units(manifest, memory) + index = {e.id: e for e in memory} + programs = {f: Program(arm_spec(manifest, manifest.arms["A"], f, store), memory, excerpt_text, vectors) for f in manifest.families} + models = {spec_of(load_program(store, manifest.programs[f])).model for f in manifest.families} + if models != {caller.model}: + raise ExperimentError(f"{manifest.id}: the programs' models {sorted(models)} differ from the caller's {caller.model!r}") + candidate_of = candidate_from_example(excerpt_text, vectors) + have = {outcome_id(manifest.id, o.arm, o.unit, o.replicate) for o in load_outcomes(store, manifest.id)} + decided, skipped = {SCORE_ARM: 0}, {"already": 0, MISS_VERDICT: 0} + fold_seed = int(manifest.folds.get("seed", 0)) + for unit in sorted(units, key=lambda u: (u.group, u.family, u.unit)): + if outcome_id(manifest.id, SCORE_ARM, unit.unit, 0) in have: + skipped["already"] += 1 + continue + candidate = candidate_of(index[unit.unit]) + if candidate is None: + raise ExperimentError(f"{manifest.id}: unit {unit.unit[:12]} has no excerpt in the store") + try: + decision = programs[unit.family].decide(candidate, by_name[unit.fold], caller, seed=fold_seed) + except BudgetExhausted: # a response is not cached: the zero ceiling stopped the call before it started + outcome = Outcome(unit.unit, unit.group, unit.family, unit.label, unit.pair, SCORE_ARM, 0, None, MISS_VERDICT, 0.0) + skipped[MISS_VERDICT] += 1 + store.put_row(outcome_row(manifest.id, str(row["pin"]), outcome)) + continue + outcome = Outcome(unit.unit, unit.group, unit.family, unit.label, unit.pair, SCORE_ARM, 0, decision.s, decision.verdict, 0.0, + decision.parse_failed) # fmt: skip + store.put_row(outcome_row(manifest.id, str(row["pin"]), outcome)) + decided[SCORE_ARM] += 1 + if on_decision is not None: + on_decision(outcome, decision) + if meter.calls or (meter.usd or 0.0) > 0 or caller.calls: + raise ZeroCostViolation(f"{manifest.id}: the meter recorded {meter.calls} calls and ${meter.usd}; the run is void") + spend = {SCORE_ARM: {"usd": meter.usd, "client_calls": meter.calls, "replayed": caller.replayed, "cache_digest": caller.cache_digest(), + "replay_priced_usd": caller.usd()}} # fmt: skip + summary = RunSummary(manifest.id, "done", len(units), decided, skipped, spend, fdigest) + store.put_row({ + "id": row_id(EXPERIMENT_KIND, f"experiment-{manifest.id}", RUN_TASK, manifest.id), "kind": EXPERIMENT_KIND, + "repo": experiment_repo(manifest.id), "pin": str(row["pin"]), "run": f"experiment-{manifest.id}", "task": RUN_TASK, + "population": POPULATION, "split": manifest.id, "image": "host", "experiment": manifest.id, + "manifest_digest": manifest.digest, **summary.as_dict(), + }) # fmt: skip + return summary + + +def min_flagged_for_bound(target: float, z: float = 1.959964) -> int: + """The fewest flagged units, all correct, whose Wilson lower bound reaches ``target`` (73 at 0.95).""" + n = 1 + while wilson_lower(n, n, z) < target: + n += 1 + return n + + +def _group_bootstrap_precision(rows: Sequence[Prediction], *, resamples: int, seed: int) -> list[float] | None: + by_group: dict[str, list[Prediction]] = {} + for r in rows: + by_group.setdefault(r.group, []).append(r) + names = sorted(by_group) + rng = random.Random(seed) + values = [] + for _ in range(resamples if names else 0): + sample = [r for g in (rng.choice(names) for _ in names) for r in by_group[g]] + flagged = [r for r in sample if r.point == "BLOCK"] + if flagged: + values.append(sum(r.label for r in flagged) / len(flagged)) + if len(values) < 2: + return None + values.sort() + return [values[int(0.025 * (len(values) - 1))], values[int(0.975 * (len(values) - 1))]] + + +def flag_summary(rows: Sequence[Prediction], target: float, *, resamples: int = 2000, seed: int = 0) -> dict[str, Any]: + """Held-out BLOCK decisions of one population: flagged count, flagged precision with its Wilson 95% lower bound + and a repository-cluster bootstrap interval, coverage (the share of positives flagged), and the M4 reading.""" + flagged = [r for r in rows if r.point == "BLOCK"] + hits = sum(r.label for r in flagged) + positives = sum(r.label for r in rows) + precision = hits / len(flagged) if flagged else None + lower = wilson_lower(hits, len(flagged)) if flagged else None + return { + "units": len(rows), "groups": len({r.group for r in rows}), "positives": positives, "flagged": len(flagged), + "true_flagged": hits, "flagged_precision": None if precision is None else round(precision, 4), + "flagged_precision_lb": None if lower is None else round(lower, 4), + "flagged_precision_ci": _group_bootstrap_precision(rows, resamples=resamples, seed=seed) if flagged else None, + "coverage": round(hits / positives, 4) if positives else None, + "meets_target_point": precision is not None and precision >= target, + "meets_target_lb": lower is not None and lower >= target, + } # fmt: skip + + +def _apply_rule(arm: Arm, preds: Sequence[Prediction], seed: int) -> tuple[list[Prediction], dict[str, Any]]: + if arm.rule["name"] == "calibrated_block": + return calibrated_block(preds, seed=seed) + select_on = str(arm.rule.get("select_on", "paired")) + decided, thresholds = nested_precision_block( + preds, eligible=(lambda r: bool(r.pair)) if select_on == "paired" else None, + precision=float(arm.rule.get("precision", TARGET_PRECISION)), bound=str(arm.rule.get("bound", "wilson")), + min_flagged=int(arm.rule.get("min_flagged", 1)), + ) # fmt: skip + return decided, {f: {"thresholds_by_fold": t} for f, t in thresholds.items()} + + +def decide_rule(manifest: Manifest, families: Mapping[str, Mapping[str, Any]]) -> tuple[str, str]: + """The pre-registered rule: **reject** the decision arm when it flags held-out units of the primary population in + some family at a point precision below the target; **adopt** it when it does not and some family's held-out + Wilson lower bound reaches the target; otherwise **inconclusive** (the incumbent rule stays).""" + decision = dict(manifest.raw["decision"]) + arm, target = str(decision["arm"]), float(decision["target_precision"]) + primary = str(manifest.raw["populations"]["primary"]) + cells = {f: v[arm][primary] for f, v in families.items()} + low = sorted(f for f, c in cells.items() if c["flagged"] and not c["meets_target_point"]) + if low: + return "reject", f"{arm} flags held-out {primary} units below precision {target} in {', '.join(low)}" + reached = sorted(f for f, c in cells.items() if c["meets_target_lb"]) + if reached: + return "adopt", f"{arm}'s held-out Wilson lower bound reaches {target} in {', '.join(reached)} and nowhere flags below it" + if not any(c["flagged"] for c in cells.values()): + return "inconclusive", f"{arm} offers no BLOCK on held-out {primary} units in any family; the incumbent stays" + return "inconclusive", f"{arm} flags only at a Wilson lower bound below {target}; the incumbent stays" + + +def analyse_rule( + manifest: Manifest, + outcomes: Sequence[Outcome], + units: Sequence[Unit], + *, + resamples: int | None = None, + provenance: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Counts and intervals only: every arm's rule applied to the same held-out scores, per family and population, + the pooled-over-families totals, the exclusions, the training capacity of each family against the Wilson bound, + and the pre-registered verdict.""" + n = resamples if resamples is not None else min(manifest.resamples, 2000) + target = float(manifest.raw["decision"]["target_precision"]) + populations = [str(manifest.raw["populations"]["primary"]), *[str(p) for p in manifest.raw["populations"].get("secondary", [])]] + by_unit = {u.unit: u for u in units} + scores = {o.unit: o for o in outcomes if o.arm == SCORE_ARM and o.replicate == 0 and o.unit in by_unit} + preds = [ + Prediction(by_unit[u].fold, o.family, o.group, o.label, float(o.s), o.verdict, parse_failed=o.parse_failed, pair=o.pair) + for u, o in sorted(scores.items()) + if o.s is not None + ] # fmt: skip + excluded = {f: sum(1 for o in scores.values() if o.s is None and o.family == f) for f in manifest.families} + not_run = {f: sum(1 for u in units if u.family == f and u.unit not in scores) for f in manifest.families} + + def select(rows: Sequence[Prediction], population: str) -> list[Prediction]: + return [r for r in rows if r.pair] if population == "paired" else list(rows) + + decided = {name: _apply_rule(arm, preds, manifest.seed) for name, arm in manifest.arms.items()} + families: dict[str, dict[str, Any]] = {} + for family in manifest.families: + cell: dict[str, Any] = {} + for name, (rows, info) in decided.items(): + mine = [r for r in rows if r.family == family] + cell[name] = {**info.get(family, {}), **{p: flag_summary(select(mine, p), target, resamples=n, seed=manifest.seed) + for p in populations}} # fmt: skip + mine = [r for r in preds if r.family == family] + folds = sorted({r.fold for r in mine}) + cell["capacity"] = { + "evaluated": len(mine), "excluded_replay_miss": excluded[family], "not_run": not_run[family], + "paired_positives_by_training_folds": {f: sum(r.label for r in mine if r.pair and r.fold != f) for f in folds}, + } # fmt: skip + families[family] = cell + overall = { + name: {p: flag_summary(select(rows, p), target, resamples=n, seed=manifest.seed) for p in populations} + for name, (rows, _) in decided.items() + } + verdict, reason = decide_rule(manifest, families) + spend = {SCORE_ARM: round(sum(o.usd or 0.0 for o in outcomes if o.arm == SCORE_ARM), 6)} + return { + "experiment": manifest.id, "manifest_digest": manifest.digest, "hypothesis": manifest.hypothesis, + "arms": {a: {"label": arm.label, "rule": dict(arm.rule)} for a, arm in manifest.arms.items()}, + "units": {"frozen": len(units), "evaluated": len(preds), "excluded_replay_miss": sum(excluded.values()), + "not_run": sum(not_run.values()), "paired_evaluated": sum(1 for r in preds if r.pair), + "groups": len({r.group for r in preds}), "by_label": {str(lab): sum(r.label == lab for r in preds) for lab in (0, 1)}, + "unsure": sum(1 for r in preds if not r.blockable)}, + "populations": {"primary": populations[0], "secondary": populations[1:]}, "target_precision": target, + "min_flagged_for_bound": min_flagged_for_bound(target), "primary": [m.as_dict() for m in manifest.primary], + "secondary": [m.as_dict() for m in manifest.secondary], "families": families, "overall": overall, "resamples": n, + "decision": verdict, "reason": reason, "spend_usd": spend, **dict(provenance or {}), + } # fmt: skip + + +def rule_result_row(manifest: Manifest, pin: str, result: Mapping[str, Any]) -> dict[str, Any]: + """The ``experiment_result`` row of a rule experiment, with the column types of :func:`result_row`.""" + primary = result["populations"]["primary"] + tests = { + f: {a: {k: cell[a][primary][k] for k in ("flagged", "flagged_precision", "flagged_precision_lb", "coverage")} + for a in manifest.arms} + for f, cell in result["families"].items() + } # fmt: skip + return { + "id": row_id(RESULT_KIND, f"experiment-{manifest.id}", "analyse", manifest.id), "kind": RESULT_KIND, + "repo": experiment_repo(manifest.id), "pin": pin, "run": f"experiment-{manifest.id}", "task": "analyse", "population": POPULATION, + "split": manifest.id, "image": "host", "experiment": manifest.id, "manifest_digest": manifest.digest, + "decision": result["decision"], "reason": result["reason"], "unit_counts": result["units"], "spend_usd": result["spend_usd"], + "tests": tests, "looks_taken": [], + } # fmt: skip + + __all__ = [ + "MISS_VERDICT", "POPULATIONS", "RULES", "RULE_METRICS", "SCORE_ARM", "ZeroCostViolation", "analyse_rule", "decide_rule", + "flag_summary", "min_flagged_for_bound", "rule_result_row", "run_rule", "zero_cost_caller", "ALPHA", "ARM_KINDS", "EXPERIMENT_KIND", "MANIFEST_DIR", "METRICS", "OBF_TWO_LOOKS", "OUTCOME_KIND", "RESAMPLES", "RESULT_DIR", "RESULT_KIND", "TOST_MARGIN", "VERDICTS", "Arm", "ArmUnit", "Estimate", "ExperimentError", "Look", "Manifest", "McNemar", "Metric", "Outcome", "RunSummary", "Unit", "analyse", "arm_spec", "arm_units", "auc", "check_registered", "committed_at_head", "decide", diff --git a/tests/test_learn_rules.py b/tests/test_learn_rules.py new file mode 100644 index 0000000..6eda3ee --- /dev/null +++ b/tests/test_learn_rules.py @@ -0,0 +1,247 @@ +"""Decision-rule experiments (kind: rule): selective blocking on the raw score, chosen on training folds only, against +today's calibrated BLOCK rule, scored once from the response cache at zero cost (exp-003).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from test_learn_experiments import CODE, EXAMPLES, PRICES, _json_shape, artifact, committed, example_rows, excerpt_text, paid, put_program + +from openultrasast.learn import experiments as ex +from openultrasast.learn.evaluate import Prediction, calibrated_block, nested_precision_block, precision_threshold, wilson_lower +from openultrasast.learn.program import Candidate, Program +from openultrasast.learn.retrieve import Target +from openultrasast.plane.memory import FileStore + +ROOT = Path(__file__).resolve().parents[1] + +RULE_MANIFEST = """ +id: {id} +hypothesis: selective blocking beats the calibrated rule +kind: rule +programs: {{injection: {pid}, path: {pid}}} +arms: + A: {{label: today, rule: {{name: calibrated_block}}}} + B: {{label: bound, rule: {{name: precision_bound_block, bound: wilson, precision: 0.95, select_on: paired}}}} + C: {{label: point, rule: {{name: precision_bound_block, bound: point, precision: 0.95, min_flagged: 2, select_on: paired}}}} +populations: {{primary: paired, secondary: [pooled]}} +decision: {{arm: B, target_precision: 0.95}} +metric: + primary: [{{name: flagged_precision_lb, target: 0.95}}] + secondary: [flagged_precision, coverage, flagged] +test: {{resamples: 50}} +budget_usd: {{total: 0}} +families: [injection, path] +folds: {{k: 5, seed: 0, compile_fraction: 0.25}} +units_file: {units} +""" + + +def write_rule_manifest(tmp_path: Path, pid: str, name: str = "exp-901-rule") -> Path: + path = tmp_path / f"{name}.yaml" + path.write_text(RULE_MANIFEST.format(id=name, pid=pid, units=tmp_path / f"{name}.units.jsonl")) + return path + + +def pred(fold: int, label: int, s: float, *, family: str = "injection", group: str = "", verdict: str = "", pair: str = "p") -> Prediction: + verdict = verdict or ("vulnerable" if s >= 0.5 else "not_vulnerable") + return Prediction(f"outer-{fold}", family, group or f"org/r{fold}", label, s, verdict, pair=pair) + + +# --- the threshold --------------------------------------------------------------------------------------------------------- + + +def test_the_wilson_bound_needs_73_clean_flags_at_095() -> None: + assert ex.min_flagged_for_bound(0.95) == 73 + assert wilson_lower(72, 72) < 0.95 <= wilson_lower(73, 73) + clean = [pred(0, 1, 0.9) for _ in range(73)] + [pred(0, 0, 0.2) for _ in range(10)] + assert precision_threshold(clean) == 0.9 + assert precision_threshold(clean[1:]) is None # 72 clean flags never reach the bound + assert precision_threshold(clean[1:], bound="point", min_flagged=10) == 0.9 + assert precision_threshold(clean[:5], bound="point", min_flagged=10) is None + + +def test_the_lowest_qualifying_threshold_and_unsure_never_blocks() -> None: + rows = [pred(0, 1, 0.9) for _ in range(80)] + [pred(0, 1, 0.4) for _ in range(5)] + assert precision_threshold(rows) == 0.4 # 85 clean flags at 0.4: the lowest threshold that qualifies + assert precision_threshold([*rows, pred(0, 0, 0.6)]) == 0.9 # one false flag at 0.6: 85 of 86 and 80 of 81 do not + unsure = [pred(0, 1, 0.95, verdict="unsure") for _ in range(100)] + assert precision_threshold(unsure) is None + decided, _ = nested_precision_block([*unsure[:3], *(pred(1, 1, 0.95) for _ in range(80))]) + assert all(r.point == "DROP" for r in decided if r.verdict == "unsure") + + +def leaky_threshold(preds: list[Prediction], fold: str) -> float | None: + """What a rule that peeked at the held-out fold would choose: every fold's rows, the reported fold included.""" + return precision_threshold([r for r in preds if r.family == "injection"]) + + +def test_no_threshold_is_chosen_with_outer_fold_labels() -> None: + """Fold outer-0 holds 80 high-scoring positives; the other folds alone never reach the bound. A rule that used + outer-0's labels would block on outer-0; the nested rule must not, and flipping outer-0's labels must not move + outer-0's threshold.""" + held = [pred(0, 1, 0.9, group=f"org/h{i}") for i in range(80)] + train = [pred(f, lab, 0.9 if lab else 0.3, group=f"org/t{f}-{i}") for f in range(1, 5) for i, lab in enumerate((1, 1, 1, 0, 0))] + preds = held + train + assert leaky_threshold(preds, "outer-0") is not None # the test has teeth: the outer labels would qualify + decided, thresholds = nested_precision_block(preds) + assert thresholds["injection"]["outer-0"] is None + assert all(r.point == "DROP" for r in decided if r.fold == "outer-0") + flipped = [Prediction(r.fold, r.family, r.group, 1 - r.label, r.s, r.verdict, pair=r.pair) if r.fold == "outer-0" else r for r in preds] + _, again = nested_precision_block(flipped) + assert again["injection"]["outer-0"] == thresholds["injection"]["outer-0"] + # and the other folds' thresholds do see outer-0 (it is their training data) + assert all(thresholds["injection"][f"outer-{f}"] == 0.9 for f in range(1, 5)) + assert all(again["injection"][f"outer-{f}"] is None for f in range(1, 5)) + + +def test_select_on_paired_ignores_unpaired_training_rows() -> None: + unpaired = [pred(1, 1, 0.9, pair="", group=f"org/u{i}") for i in range(80)] + paired = [pred(1, lab, 0.9 if lab else 0.1, group=f"org/p{i}") for i, lab in enumerate((1, 0) * 3)] + held = [pred(0, 1, 0.9), pred(0, 0, 0.1)] + _, everyone = nested_precision_block([*unpaired, *paired, *held]) + _, only_paired = nested_precision_block([*unpaired, *paired, *held], eligible=lambda r: bool(r.pair)) + assert everyone["injection"]["outer-0"] == 0.9 and only_paired["injection"]["outer-0"] is None + + +def test_the_calibrated_rule_blocks_only_where_calibration_holds(monkeypatch: pytest.MonkeyPatch) -> None: + rows = [pred(f, 1 if i < 30 else 0, 0.9 if i < 30 else 0.1, group=f"org/g{f}-{i}") for f in range(5) for i in range(40)] + import openultrasast.learn.evaluate as ev + + real = ev.check + for holds in (False, True): + monkeypatch.setattr(ev, "check", lambda *a, _h=holds, **k: ev.replace(real(*a, **k), holds=_h)) + decided, info = calibrated_block(rows) + assert info["injection"]["calibration_holds"] is holds + blocked = [r for r in decided if r.point == "BLOCK"] + assert bool(blocked) is holds and all(r.label == 1 for r in blocked) + + +# --- the manifest ------------------------------------------------------------------------------------------------------------ + + +def test_rule_manifests_cost_nothing_and_change_only_the_rule(tmp_path: Path) -> None: + path = write_rule_manifest(tmp_path, "0" * 64) + manifest = ex.load_manifest(path) + assert manifest.kind == "rule" and manifest.arms["B"].rule["bound"] == "wilson" and manifest.budget_total_usd == 0 + text = path.read_text() + for bad, why in ( + (text.replace("budget_usd: {total: 0}", "budget_usd: {total: 1}"), "budget_usd must be 0"), + (text.replace("A: {label: today, ", "A: {label: today, spec: {sampling: retrieval_ensemble}, "), "BLOCK rule only"), + (text.replace("name: calibrated_block", "name: magic"), "rule.name"), + (text.replace("bound: wilson", "bound: hope"), "bound in"), + (text.replace("primary: paired", "primary: all"), "populations"), + (text.replace("arm: B,", "arm: Z,"), "decision"), + (text.replace("flagged_precision_lb", "auc"), "metric"), + (text.replace("programs: {injection: " + "0" * 64 + ", path: ", "programs: {path: "), "program for every family"), + ): + path.write_text(bad) + with pytest.raises(ex.ExperimentError, match=why): + ex.load_manifest(path) + + +# --- the zero-cost run ------------------------------------------------------------------------------------------------------ + + +def prepared(tmp_path: Path) -> tuple[FileStore, ex.Manifest, list[ex.Unit]]: + store = FileStore(tmp_path / "memory") + store.put_rows(example_rows()) + pid = put_program(store, artifact()) + manifest = ex.load_manifest(write_rule_manifest(tmp_path, pid)) + units = ex.freeze_units(store, manifest, EXAMPLES) + ex.write_units(manifest.units_file, units) + ex.register(store, manifest, commit_of=committed) + return store, manifest, units + + +def test_the_run_replays_the_cache_excludes_misses_and_proves_zero_spend(tmp_path: Path) -> None: + store, manifest, units = prepared(tmp_path) + chat, caller = paid() + caller.store = store + folds = {f.name: f for f in ex.program_folds(EXAMPLES, manifest)[0]} + index = {e.id: e for e in EXAMPLES} + cached = units[: len(units) - 3] # the incumbent's evaluation answered all but three units + for u in cached: + e = index[u.unit] + program = Program(ex.arm_spec(manifest, manifest.arms["A"], u.family, store), EXAMPLES, excerpt_text) + program.decide(Candidate(Target.of(e), CODE[e.excerpt_sha], "python"), folds[u.fold], caller, seed=0) + paid_calls = len(chat.calls) + with pytest.raises(ex.ZeroCostViolation): # a paid caller is refused before anything is decided + ex.run_rule(store, manifest, EXAMPLES, caller, caller.client, excerpt_text) + zero, meter = ex.zero_cost_caller("deepseek-flash", PRICES, store) + summary = ex.run_rule(store, manifest, EXAMPLES, zero, meter, excerpt_text) + assert len(chat.calls) == paid_calls and meter.calls == 0 and meter.usd == 0.0 and zero.calls == 0 + assert summary.decided == {"score": len(cached)} and summary.skipped == {"already": 0, "replay_miss": 3} + assert summary.spend["score"]["client_calls"] == 0 and summary.spend["score"]["usd"] == 0.0 + outcomes = ex.load_outcomes(store, manifest.id) + assert sum(o.verdict == ex.MISS_VERDICT and o.s is None for o in outcomes) == 3 + report = ex.analyse_rule(manifest, outcomes, units, resamples=20, provenance={"code_commit": "c" * 40}) + assert report["units"]["excluded_replay_miss"] == 3 and report["units"]["evaluated"] == len(cached) + assert report["spend_usd"] == {"score": 0.0} and report["decision"] in ex.VERDICTS + assert set(report["families"]["injection"]) == {"A", "B", "C", "capacity"} and report["min_flagged_for_bound"] == 73 + text = json.dumps(report) + assert "org0/repo0" not in text and units[0].unit not in text # counts only + store.put_row(ex.rule_result_row(manifest, "c" * 40, report)) + shapes: dict[str, set[str]] = {} # one JSON type per column across the experiment's rows (S3 Select, see test_learn_experiments) + for record in store.rows(repo=ex.experiment_repo(manifest.id)): + for name, value in record.row.items(): + if value is not None: + shapes.setdefault(name, set()).add(_json_shape(value)) + assert {name: sorted(s) for name, s in shapes.items() if len(s) > 1} == {} + again = ex.run_rule(store, manifest, EXAMPLES, *ex.zero_cost_caller("deepseek-flash", PRICES, store), excerpt_text) + assert again.decided == {"score": 0} and again.skipped["already"] == len(units) + with pytest.raises(ex.ExperimentError, match="zero cost"): + ex.run(store, manifest, EXAMPLES, {}, excerpt_text) + + +def test_a_call_past_the_meter_voids_the_run() -> None: + caller, meter = ex.zero_cost_caller("deepseek-flash", PRICES, None) + with pytest.raises(ex.ZeroCostViolation): + meter.inner.complete(model="deepseek-flash", messages=[], tools=[]) + + +# --- the analysis -------------------------------------------------------------------------------------------------------------- + + +def crafted(tmp_path: Path, *, b_clean: bool) -> tuple[ex.Manifest, list[ex.Outcome], list[ex.Unit]]: + """Five folds of 20 candidate pairs each: positives at s 0.9; negatives at 0.1, except four per fold at 0.9 unless + ``b_clean``.""" + manifest = ex.load_manifest(write_rule_manifest(tmp_path, "0" * 64)) + units, outcomes = [], [] + for f in range(5): + for i in range(40): + label = 1 if i < 20 else 0 + high = label == 1 or (not b_clean and i >= 36) + u = ex.Unit(f"u{f}-{i}", f"org/r{f}-{i % 20}", "injection", label, f"outer-{f}", f"p{f}-{i % 20}") + units.append(u) + s = 0.9 if high else 0.1 + outcomes.append(ex.Outcome(u.unit, u.group, u.family, label, u.pair, ex.SCORE_ARM, 0, s, + "vulnerable" if high else "not_vulnerable", 0.0)) # fmt: skip + return manifest, outcomes, units + + +def test_the_verdict_follows_the_registered_rule(tmp_path: Path) -> None: + manifest, outcomes, units = crafted(tmp_path, b_clean=True) + report = ex.analyse_rule(manifest, outcomes, units, resamples=20) + cell = report["families"]["injection"]["B"]["paired"] + assert cell["flagged"] == 100 and cell["flagged_precision"] == 1.0 and cell["coverage"] == 1.0 and cell["meets_target_lb"] + assert report["decision"] == "adopt" + assert report["families"]["path"]["B"]["paired"]["flagged"] == 0 # a family without units flags nothing + manifest, outcomes, units = crafted(tmp_path, b_clean=False) + report = ex.analyse_rule(manifest, outcomes, units, resamples=20) + assert report["families"]["injection"]["B"]["paired"]["flagged"] == 0 and report["decision"] == "inconclusive" + assert report["families"]["injection"]["C"]["paired"]["flagged"] == 0 # 80/96 point precision never reaches 0.95 either + row = ex.rule_result_row(manifest, "c" * 40, report) + assert row["kind"] == "experiment_result" and row["tests"]["injection"]["B"]["flagged"] == 0 and row["looks_taken"] == [] + + +def test_the_registered_exp003_manifest_loads() -> None: + path = ROOT / "plane/experiments/exp-003-precision-bound-blocking.yaml" + if not path.exists(): + pytest.skip("exp-003 is not written yet") + manifest = ex.load_manifest(path) + assert manifest.kind == "rule" and manifest.budget_total_usd == 0 and manifest.raw["populations"]["primary"] == "paired" + assert manifest.arms["A"].rule == {"name": "calibrated_block"} and manifest.arms["B"].rule["bound"] == "wilson" + assert len(manifest.families) == 6 and set(manifest.programs) == set(manifest.families)