From da91ef0dd9dcbbf65259dc453ac4361660d07f7b Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:47:20 +0200 Subject: [PATCH] exp-003: supplementary all-units replay after the cache fill; exp-004 registered Supplementary analysis, not a new decision: exp-003's verdict stands. The 142 units excluded for an uncached response were filled with the same six incumbent programs, parameters and request keys, on exp-003's 937-example memory (the harvest's 196 rows and its 2026-10-03 embeddings left out; a zero-cost replay first reproduced exp-003 exactly: 211 decided, 1056 replayed, $0.582876). Fill: MeteredClient, ceiling $0.99 under the approved $1.00, 697 client calls, $0.331759, no provider error. Replay on all 353 units (244 paired, 146 groups): 0 client calls, $0.0, 1768 responses replayed; no arm flags any unit, the registered rule reads inconclusive. Calibration holds only for untrusted destination (ECE 0.0454) and arm A still finds no BLOCK point there. benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json exp-004 registered in s3://sast-memory (manifest 87fceb65..., pin 2ad9c1d, units pending): benchmarks/experiments/exp-004-pooled-block-gate/ registration.json. docs/decision-engine.md: both, briefly. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .../result-complete-cache.json | 1081 +++++++++++++++++ .../registration.json | 35 + docs/decision-engine.md | 41 +- 3 files changed, 1156 insertions(+), 1 deletion(-) create mode 100644 benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json create mode 100644 benchmarks/experiments/exp-004-pooled-block-gate/registration.json diff --git a/benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json b/benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json new file mode 100644 index 0000000..b8eeba6 --- /dev/null +++ b/benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json @@ -0,0 +1,1081 @@ +{ + "arms": { + "A": { + "label": "today's rule -- Platt map cross-fitted per family, BLOCK only when the calibration check holds, BLOCK point nested on the other folds' calibrated p", + "rule": { + "name": "calibrated_block" + } + }, + "B": { + "label": "selective blocking -- per-family raw-score threshold from the other folds' paired units, lowest t with Wilson 95% LB of flagged precision >= 0.95", + "rule": { + "bound": "wilson", + "name": "precision_bound_block", + "precision": 0.95, + "select_on": "paired" + } + }, + "C": { + "label": "secondary, non-deciding -- as B with point precision >= 0.95 over at least 10 training flags", + "rule": { + "bound": "point", + "min_flagged": 10, + "name": "precision_bound_block", + "precision": 0.95, + "select_on": "paired" + } + } + }, + "code_commit": "5b2bd9bb91e467efd33630db7c6c827e29ed21a5", + "code_note": "run on 5b2bd9b's working tree with the exp-004 additions to learn/evaluate.py and learn/experiments.py (committed on this branch); they add the pooled_block_gate rule and leave a manifest without it analysed as before", + "decision": "inconclusive", + "exp003_result": "benchmarks/experiments/exp-003-precision-bound-blocking/result.json", + "experiment": "exp-003-precision-bound-blocking", + "families": { + "access_control": { + "A": { + "calibration_holds": false, + "ece": 0.077, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 24, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 41 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 24, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 41 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 24, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 25, + "true_flagged": 0, + "units": 41 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 41, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 12, + "outer-1": 13, + "outer-2": 13, + "outer-3": 13, + "outer-4": 13 + } + } + }, + "deserialization": { + "A": { + "calibration_holds": false, + "ece": 0.0596, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 17, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 17, + "true_flagged": 0, + "units": 34 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 31, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 31, + "true_flagged": 0, + "units": 48 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 17, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 17, + "true_flagged": 0, + "units": 34 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 31, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 31, + "true_flagged": 0, + "units": 48 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 17, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 17, + "true_flagged": 0, + "units": 34 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 31, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 31, + "true_flagged": 0, + "units": 48 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 48, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 14, + "outer-1": 14, + "outer-2": 13, + "outer-3": 13, + "outer-4": 14 + } + } + }, + "injection": { + "A": { + "calibration_holds": false, + "ece": 0.1067, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 26, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 76 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 34, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 63, + "true_flagged": 0, + "units": 119 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 26, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 76 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 34, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 63, + "true_flagged": 0, + "units": 119 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 26, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 38, + "true_flagged": 0, + "units": 76 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 34, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 63, + "true_flagged": 0, + "units": 119 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 119, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 29, + "outer-1": 33, + "outer-2": 33, + "outer-3": 26, + "outer-4": 31 + } + } + }, + "output_encoding": { + "A": { + "calibration_holds": false, + "ece": 0.0761, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 27, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 29, + "true_flagged": 0, + "units": 45 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 27, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 29, + "true_flagged": 0, + "units": 45 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 27, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 29, + "true_flagged": 0, + "units": 45 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 45, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 12, + "outer-1": 13, + "outer-2": 13, + "outer-3": 13, + "outer-4": 13 + } + } + }, + "path": { + "A": { + "calibration_holds": false, + "ece": 0.062, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 23, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 23, + "true_flagged": 0, + "units": 39 + }, + "reason_by_fold": { + "outer-0": "calibration", + "outer-1": "calibration", + "outer-2": "calibration", + "outer-3": "calibration", + "outer-4": "calibration" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 23, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 23, + "true_flagged": 0, + "units": 39 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 16, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 16, + "true_flagged": 0, + "units": 32 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 23, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 23, + "true_flagged": 0, + "units": 39 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 39, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 12, + "outer-1": 13, + "outer-2": 13, + "outer-3": 13, + "outer-4": 13 + } + } + }, + "untrusted_destination": { + "A": { + "calibration_holds": true, + "ece": 0.0454, + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 18, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 19, + "true_flagged": 0, + "units": 38 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 36, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 37, + "true_flagged": 0, + "units": 61 + }, + "reason_by_fold": { + "outer-0": "precision_unreachable", + "outer-1": "precision_unreachable", + "outer-2": "precision_unreachable", + "outer-3": "precision_unreachable", + "outer-4": "precision_unreachable" + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 18, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 19, + "true_flagged": 0, + "units": 38 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 36, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 37, + "true_flagged": 0, + "units": 61 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 18, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 19, + "true_flagged": 0, + "units": 38 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 36, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 37, + "true_flagged": 0, + "units": 61 + }, + "thresholds_by_fold": { + "outer-0": null, + "outer-1": null, + "outer-2": null, + "outer-3": null, + "outer-4": null + } + }, + "capacity": { + "evaluated": 61, + "excluded_replay_miss": 0, + "not_run": 0, + "paired_positives_by_training_folds": { + "outer-0": 16, + "outer-1": 15, + "outer-2": 15, + "outer-3": 15, + "outer-4": 15 + } + } + } + }, + "fill": { + "approved_ceiling_usd": 1.0, + "by_step": { + "fill": { + "client_calls": 692, + "replayed_within_fill": 20, + "units": 141, + "usd": 0.329427 + }, + "smoke_test": { + "client_calls": 5, + "units": 1, + "usd": 0.002332 + } + }, + "client_calls": 697, + "consistency": "replay priced usd 0.914636 = exp-003's 0.582876 + this fill's 0.331759", + "date": "2026-10-03", + "meter": "plane.budget.MeteredClient around the DeepSeek chat endpoint (plane/models/deepseek-flash.yaml prices), checked before every call; the same request keys as exp-003 (learn.program.Caller, the six incumbent programs, fold seed 0)", + "meter_ceiling_usd": 0.99, + "provider_errors": 0, + "units": 142, + "usd": 0.331759 + }, + "hypothesis": "The calibrated-probability BLOCK rule (Platt map + calibration gate, ECE <= 0.05) is the wrong instrument for M4, which asks for precision on what is flagged. Selective blocking -- a per-family raw-score threshold chosen on training folds only as the lowest one whose Wilson 95% lower bound on flagged precision is >= 0.95 -- reaches the M4 precision bar on held-out repositories where the calibrated rule offers no BLOCK.\n", + "instrument_check": { + "families": { + "access_control": { + "distinct_s": 30, + "paired": { + "blockable": 28, + "positives_at_or_above_first_negative": 10, + "positives_before_first_negative": 10 + }, + "pooled": { + "blockable": 37, + "positives_at_or_above_first_negative": 19, + "positives_before_first_negative": 19 + }, + "verdicts": { + "not_vulnerable": 19, + "unsure": 4, + "vulnerable": 18 + } + }, + "deserialization": { + "distinct_s": 28, + "paired": { + "blockable": 29, + "positives_at_or_above_first_negative": 7, + "positives_before_first_negative": 7 + }, + "pooled": { + "blockable": 43, + "positives_at_or_above_first_negative": 21, + "positives_before_first_negative": 21 + }, + "verdicts": { + "not_vulnerable": 15, + "unsure": 5, + "vulnerable": 28 + } + }, + "injection": { + "distinct_s": 48, + "paired": { + "blockable": 50, + "positives_at_or_above_first_negative": 3, + "positives_before_first_negative": 2 + }, + "pooled": { + "blockable": 77, + "positives_at_or_above_first_negative": 14, + "positives_before_first_negative": 8 + }, + "verdicts": { + "not_vulnerable": 46, + "unsure": 42, + "vulnerable": 31 + } + }, + "output_encoding": { + "distinct_s": 28, + "paired": { + "blockable": 18, + "positives_at_or_above_first_negative": 0, + "positives_before_first_negative": 0 + }, + "pooled": { + "blockable": 31, + "positives_at_or_above_first_negative": 13, + "positives_before_first_negative": 13 + }, + "verdicts": { + "not_vulnerable": 16, + "unsure": 14, + "vulnerable": 15 + } + }, + "path": { + "distinct_s": 27, + "paired": { + "blockable": 21, + "positives_at_or_above_first_negative": 8, + "positives_before_first_negative": 8 + }, + "pooled": { + "blockable": 28, + "positives_at_or_above_first_negative": 15, + "positives_before_first_negative": 15 + }, + "verdicts": { + "not_vulnerable": 13, + "unsure": 11, + "vulnerable": 15 + } + }, + "untrusted_destination": { + "distinct_s": 31, + "paired": { + "blockable": 20, + "positives_at_or_above_first_negative": 5, + "positives_before_first_negative": 5 + }, + "pooled": { + "blockable": 39, + "positives_at_or_above_first_negative": 22, + "positives_before_first_negative": 22 + }, + "verdicts": { + "not_vulnerable": 16, + "unsure": 22, + "vulnerable": 23 + } + } + }, + "note": "descriptive, all folds together, ranking by raw score s among non-unsure verdicts; not nested, chooses nothing; a positive tied with the highest-scored negative is not counted; exp-003's ad hoc check did not record its tie rule, so both are given; for injection exp-003 reported 3 (paired) and 11 (pooled)" + }, + "manifest_digest": "5917f9a12035f09eae76978f3b5827b9fe59c8a9a5275f9ab54c3c1b55e8db3b", + "memory_reconstruction": "exp-003 ran on the 937-example static memory. The negatives harvest (5b2bd9b) later added 196 rows (98 advisory-fixes-2 repositories) and 223 embeddings dated 2026-10-03 (183 new excerpts and 40 old ones). Both were left out here: memory 937, folds digest a1f78e14... equal to exp-003's, 788 vectors. A zero-cost replay before the fill reproduced exp-003 exactly: 211 units decided, 142 excluded, 1056 responses replayed, $0.582876 priced.", + "meter": { + "cache_digest": "df7e8169fa0205d6623897f285711f220e0ea1e379ca086896a9e8c3ec11a85f", + "client_calls": 0, + "metered_usd": 0.0, + "replay_priced_usd_not_spent": 0.914636, + "replayed_responses": 1768 + }, + "min_flagged_for_bound": 73, + "overall": { + "A": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 109, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 122, + "true_flagged": 0, + "units": 244 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 146, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 208, + "true_flagged": 0, + "units": 353 + } + }, + "B": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 109, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 122, + "true_flagged": 0, + "units": 244 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 146, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 208, + "true_flagged": 0, + "units": 353 + } + }, + "C": { + "paired": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 109, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 122, + "true_flagged": 0, + "units": 244 + }, + "pooled": { + "coverage": 0.0, + "flagged": 0, + "flagged_precision": null, + "flagged_precision_ci": null, + "flagged_precision_lb": null, + "groups": 146, + "meets_target_lb": false, + "meets_target_point": false, + "positives": 208, + "true_flagged": 0, + "units": 353 + } + } + }, + "populations": { + "primary": "paired", + "secondary": [ + "pooled" + ] + }, + "primary": [ + { + "name": "flagged_precision_lb", + "target": 0.95 + } + ], + "reason": "B offers no BLOCK on held-out paired units in any family; the incumbent stays", + "resamples": 2000, + "secondary": [ + { + "name": "flagged_precision" + }, + { + "name": "coverage" + }, + { + "name": "flagged" + } + ], + "spend_usd": { + "score": 0.0 + }, + "supplementary": "SUPPLEMENTARY ANALYSIS, NOT A NEW DECISION. exp-003's registered verdict (inconclusive, result.json, 211 of 353 units) stands. This file re-runs the same registered analysis on all 353 frozen units after the 142 units excluded for an uncached response had their responses filled (maintainer-approved spend, 2026-10-03). Its `decision` field is what the registered rule reads on the complete cache; it changes nothing.", + "target_precision": 0.95, + "units": { + "by_label": { + "0": 145, + "1": 208 + }, + "evaluated": 353, + "excluded_replay_miss": 0, + "frozen": 353, + "groups": 146, + "not_run": 0, + "paired_evaluated": 244, + "unsure": 98 + } +} diff --git a/benchmarks/experiments/exp-004-pooled-block-gate/registration.json b/benchmarks/experiments/exp-004-pooled-block-gate/registration.json new file mode 100644 index 0000000..dfd9134 --- /dev/null +++ b/benchmarks/experiments/exp-004-pooled-block-gate/registration.json @@ -0,0 +1,35 @@ +{ + "experiment": "exp-004-pooled-block-gate", + "registered": "2026-10-03", + "store": "s3://sast-memory", + "manifest": "plane/experiments/exp-004-pooled-block-gate.yaml", + "manifest_digest": "87fceb6580c6de629d6a6089eb5888cdfc4df0f791576184f987c14c192d85f5", + "pin": "2ad9c1db7ae673c2a2e48a0afadcaa795dfb1ced", + "units_status": "pending", + "units_file": "plane/experiments/exp-004-pooled-block-gate.units.jsonl", + "units_digest": "", + "units": "frozen at run time by `ousast learn experiment units` from the memory after the negatives harvest (merged in 5b2bd9b, 1133 static examples), committed, and registered again before the run; until then the run refuses (check_registered: the units file does not exist)", + "requirement": "learned-decision-engine Req 6.4, amended 2026-10-03 by maintainer decision (refines the BLOCK rule of Req 7.3 and 7.4)", + "kind": "rule: one zero-cost scoring pass of the six slice-2 incumbent programs; arms differ only in the BLOCK rule", + "zero_cost": "the run replays the response cache behind a zero-ceiling meter; a unit with any uncached response is excluded and counted. The harvest changed the memory the programs retrieve from, so a paid cache fill, approved separately with its own ceiling, must precede the run", + "normalisation": { + "choice": "per-family Platt scaling: learn.calibrate.fit_map with its existing floors (own slope and intercept per family at >= 20 positive and negative repository groups, a pooled slope with a per-family intercept at >= 10, no map below that and so no BLOCK for that family on that fold), fitted on the training folds' paired units only", + "why": "a rank-to-quantile map gives every family the same flag rate whatever its precision, so one pooled threshold would flag the top slice of a weak family as readily as of a strong one; Platt puts each family on the scale of its own training precision, which a pooled precision bound needs. The map is a normaliser only: precision is shown by the held-out bound, not by calibration (exp-003: ECE <= 0.05 does not hold per family)" + }, + "arms": { + "A": "today's rule: Platt map cross-fitted per family; BLOCK only when the calibration check holds; BLOCK point nested on the other folds' calibrated p (Wilson LB >= 0.95)", + "B": "pooled gate: per outer fold j, training units normalised by an inner cross-fit; t_block = the lowest normalised score whose Wilson 95% LB of precision over the training paired flags of all families together is >= 0.95 (unsure never flags); t_quiet per family = the highest normalised score keeping >= 90% of that family's training paired positives; fold j's units BLOCK at p >= t_block, ADVISORY at p >= t_quiet (or without a map or t_quiet), silent below" + }, + "family_floor": { + "F": 10, + "rule": "a family may block only if its own held-out paired flags (all five folds) number >= 10 with zero errors; otherwise its flags are reported as ADVISORY. The floor uses held-out labels to decide which families block and moves no threshold", + "why": "zero errors in 10 flags is unlikely for a family much worse than the bar (probability 0.056 at precision 0.75, 0.107 at 0.80); a larger F is out of reach of the smaller families (16-19 paired positives in exp-003's units over all folds); 10 is exp-003 arm C's training minimum" + }, + "primary_metric": "held-out (outer-fold) Wilson 95% lower bound of flagged precision over arm B's paired flags of all families together, before the family floor (so dropping a failing family cannot raise it)", + "populations": {"primary": "paired (both sides of the candidate present)", "secondary": ["pooled (every unit)"]}, + "secondary_metrics": ["per-family flagged precision and Wilson LB", "the family-floor table", "quiet_recall per family (held-out positives at or above t_quiet / positives) against 0.90", "after-floor totals (what users would get)", "repository-cluster bootstrap 95% CI of flagged precision", "coverage", "flagged"], + "decision_rule": "on arm B's held-out paired flags of all families together, before the floor: REJECT B if their point precision < 0.95; else ADOPT B if their Wilson 95% LB >= 0.95 and at least one family passes the floor (>= 10 held-out paired flags, zero errors) -- those families block, every other family stays advisory; else INCONCLUSIVE (the incumbent rule stays)", + "power": "the pooled bound needs >= 73 flags with no error (110 with one, 142 with two). On exp-003's units (all 353 scored, result-complete-cache.json) few paired positives rank above the first negative per family on the raw score (access control 10, deserialization 7, injection 2, output encoding 0, path 8, untrusted destination 5); normalisation keeps each family's order, so the pooled top set reaches 73 only if the harvest's pairs separate better. An inconclusive verdict is a plausible outcome", + "tests": "tests/test_learn_rules.py: test_one_pooled_threshold_blocks_where_no_family_alone_reaches_the_bound, test_no_held_out_label_moves_a_threshold_or_the_map, test_a_family_with_one_held_out_error_cannot_block, test_quiet_threshold_keeps_ninety_percent_of_positives_and_nothing_is_silenced_blind, test_the_pooled_verdict_follows_the_registered_rule, test_the_exp004_manifest_registers_the_procedure_before_its_units", + "status": "registered 2026-10-03, units pending, not run" +} diff --git a/docs/decision-engine.md b/docs/decision-engine.md index d62e064..6186e35 100644 --- a/docs/decision-engine.md +++ b/docs/decision-engine.md @@ -141,7 +141,7 @@ Changes to prompts, models, pass counts, features or sampling are compared by co experiments. Each experiment comes from a pre-registered manifest (`plane/experiments/.yaml`). This way nothing is adopted on one run's number. -Three experiments are registered: +Four experiments are registered: - exp-001 (known callers) has not run. - exp-002 compared a retrieval ensemble against today's temperature sampling. It ran on 160 @@ -150,6 +150,7 @@ Three experiments are registered: 0.875 against 0.906 and an interval spanning zero (`benchmarks/experiments/exp-002-retrieval-ensemble/result.json`). - exp-003 compared two BLOCK rules on the same scores. It was **inconclusive**. Neither rule flags anything. +- exp-004 tests one pooled BLOCK threshold with a per-family floor. It is registered and has not run. ### exp-003: precision-bound blocking (2026-10-03) @@ -196,6 +197,44 @@ Injection answered `unsure` for 42 of 119 units, and `unsure` never blocks. Record: `benchmarks/experiments/exp-003-precision-bound-blocking/result.json`. +**Supplementary: all 353 units (2026-10-03).** This is not a new decision; exp-003's verdict stands. +The 142 excluded units had their responses filled with the same programs and request keys. +The fill cost 697 client calls and $0.331759 under a $1.00 ceiling. +The replay then decided all 353 units at zero cost: 244 paired, 146 repository groups. + +| Family | Paired units (positives) | Flagged A / B / C | ECE (arm A) | +| --- | --- | --- | --- | +| access control | 32 (16) | 0 / 0 / 0 | 0.077 | +| deserialization | 34 (17) | 0 / 0 / 0 | 0.0596 | +| injection | 76 (38) | 0 / 0 / 0 | 0.1067 | +| output encoding | 32 (16) | 0 / 0 / 0 | 0.0761 | +| path | 32 (16) | 0 / 0 / 0 | 0.062 | +| untrusted destination | 38 (19) | 0 / 0 / 0 | 0.0454 | + +Still no arm flags any unit, so the registered rule reads inconclusive. +Calibration holds only for untrusted destination. +There arm A still finds no BLOCK point that reaches the bound. +Record: `benchmarks/experiments/exp-003-precision-bound-blocking/result-complete-cache.json`. + +### exp-004: pooled block gate with a family floor (registered, not run) + +The maintainer amended the BLOCK rule on 2026-10-03 (requirements, Req 6.4). +Precision is now shown across all families together, not per family. +Each family's score is first mapped by its own Platt fit, on training folds. +One threshold, t_block, then serves every family. +It is the lowest one whose pooled Wilson lower bound reaches 0.95 on training folds. +A family blocks only with at least 10 held-out flags and no error. +Otherwise its flags stay advisory. +Below a per-family t_quiet, which keeps 90% of positives, findings are silent. +No held-out label moves a threshold. + +- **Primary metric:** the pooled held-out Wilson lower bound of flagged precision, paired units, before the floor. +- **Decision:** reject below point precision 0.95; adopt if the bound reaches 0.95 and a family passes the floor. +- **Units:** frozen at run time, from the memory after the negatives harvest. +- **Cost:** the run replays the cache. The new memory needs a paid cache fill first. + +Manifest: `plane/experiments/exp-004-pooled-block-gate.yaml`. + The reading is on [Where we stand](where-we-stand.md#2-our-approach-to-detection-step-by-step). ## Commands