diff --git a/.kiro/specs/learned-decision-engine/tasks.md b/.kiro/specs/learned-decision-engine/tasks.md index 1d756d1..3eb62ac 100644 --- a/.kiro/specs/learned-decision-engine/tasks.md +++ b/.kiro/specs/learned-decision-engine/tasks.md @@ -139,8 +139,18 @@ a model or embedding call. on them, so `ousast learn experiment units` could not freeze the candidates. - [ ] 7.3 exp-001 run and recorded (~$7.7 expected, ceiling $15) -- an inconclusive result is pre-registered as likely; needs the frozen units file first. - - [ ] 7.4 exp-002 run and recorded **(budget go-ahead: ~$2 expected, ceiling $4)**; further program variants - (`k = 5` vs `k = 1`, the confidence source) as new manifests. + - [x] 7.4 exp-002 run and recorded 2026-10-02 (budget go-ahead ~$2 expected, ceiling $4): **reject B**. 160 paired + units (injection 119, access_control 41; 56 repositories; 108 units in 54 candidate pairs), both arms, two + replicates each, no early stop. Primary: canary agreement A 0.875 -> B 0.906 (diff +0.03, CI [-0.03, +0.10], + spans; B meets the 0.90 target), within-pair AUC A 0.815 -> B 0.741 (diff -0.07, CI [-0.13, -0.02], **against B**). + Secondary: AUC -0.02 (CI [-0.04, -0.003], against), ADVISORY recall -0.06 (spans; McNemar 7/2, p = 0.18), + precision +0.01 (spans), usd per candidate A $0.0021 -> B $0.0051 (CI [+0.0027, +0.0032], against: the five + disjoint example sets share only the prompt prefix in DeepSeek's cache). Priced usage A $0.68 (1206 paid calls, + 395 replayed), B $1.63 (1350 paid, 250 replayed). Result: `benchmarks/experiments/exp-002-retrieval-ensemble/result.json`, + `benchmarks/measurements/2026-10-02-exp-002/record.json`. Three repairs on the way (each its own commit): example + rows' null plane-only features (1f525ff), the 788 embedding vectors absent from RustFS (copied from the file + store), the result row's column-type collision under S3 Select (780caaa). Further program variants (`k = 5` vs + `k = 1`, the confidence source) as new manifests. - _Requirements: 3.6, 4.1-4.4_ - [ ] 8. Integration diff --git a/benchmarks/experiments/exp-002-retrieval-ensemble/registration.json b/benchmarks/experiments/exp-002-retrieval-ensemble/registration.json index 83a148f..09860b1 100644 --- a/benchmarks/experiments/exp-002-retrieval-ensemble/registration.json +++ b/benchmarks/experiments/exp-002-retrieval-ensemble/registration.json @@ -19,5 +19,5 @@ "note": "frozen on the 937-example static memory (slice 2 evaluated 110 + 37 candidates on the 904-example memory of the same day; the recovered negatives of ef6945e moved the seeded compile split and the outer folds). Freezing needed two repairs first: example rows carried the plane-only features as null in x and the renderer refused them (1f525ff), and the RustFS store held none of the 788 embedding vectors the file store had (copied over with put_blob, no embedding call)." }, "first_registration": {"registered": "2026-10-02", "pin": "fbc3220bbe1c8a4f8afa0a724fd41a51b390dd22", "units_status": "pending"}, - "status": "registered, units frozen; run pending" + "status": "run and analysed 2026-10-02: reject (result.json; benchmarks/measurements/2026-10-02-exp-002/record.json)" } diff --git a/benchmarks/experiments/exp-002-retrieval-ensemble/result.json b/benchmarks/experiments/exp-002-retrieval-ensemble/result.json new file mode 100644 index 0000000..2e8c0ac --- /dev/null +++ b/benchmarks/experiments/exp-002-retrieval-ensemble/result.json @@ -0,0 +1,241 @@ +{ + "alpha": 0.05, + "arms": { + "A": { + "budget_usd": 1.0, + "label": "today's sampling -- k = 5, sample 1 at temperature 0 and samples 2..5 at 0.7, one retrieved example set", + "repeats": 2, + "replay_only": false + }, + "B": { + "budget_usd": 3.0, + "label": "retrieval ensemble -- k = 5 at temperature 0, each sample over a disjoint retrieved example set", + "repeats": 2, + "replay_only": false + } + }, + "code_commit": "780caaa558f2cc091fd6b0fd5c618c3b488030c2", + "decision": "reject", + "experiment": "exp-002-retrieval-ensemble", + "hypothesis": "Drawing the k = 5 samples from five disjoint retrieved example sets at temperature 0 (a retrieval ensemble) makes the decision more repeatable than five temperature samples over one example set, without hurting the ranking within a candidate pair, at no higher cost per candidate.\n", + "looks": [ + { + "boundary": 2.797, + "crossed": false, + "estimate": { + "a": 0.8611111111111112, + "b": 0.8888888888888888, + "ci": [ + -0.08333333333333326, + 0.1325301204819277 + ], + "diff": 0.02777777777777768, + "groups": 28, + "metric": "canary_agreement", + "resamples": 10000, + "se": 0.05449719877526111, + "units": 72 + }, + "fraction": 0.5, + "groups": 28, + "z": 0.5097101943226363 + }, + { + "boundary": 1.977, + "crossed": false, + "estimate": { + "a": 0.875, + "b": 0.90625, + "ci": [ + -0.03448275862068961, + 0.0972972972972973 + ], + "diff": 0.03125, + "groups": 56, + "metric": "canary_agreement", + "resamples": 10000, + "se": 0.0337315848833629, + "units": 160 + }, + "fraction": 1.0, + "groups": 56, + "z": 0.9264314175588332 + } + ], + "manifest_digest": "f5004fb8618b1f7cc7dcf298d42b4ac1b6c0c1aa72a97299c77f937bd6175148", + "mde": 0.1, + "primary": [ + { + "name": "canary_agreement", + "target": 0.9 + }, + { + "name": "within_pair_auc" + } + ], + "reason": "the CI of within_pair_auc excludes 0 against B", + "resamples": 10000, + "secondary": [ + { + "name": "usd_per_candidate" + }, + { + "name": "auc" + }, + { + "name": "advisory_recall" + }, + { + "name": "advisory_precision" + } + ], + "spend_usd": { + "A": 0.682094, + "B": 1.632737 + }, + "tests": { + "advisory_precision": { + "ci_vs_zero": "spans", + "direction": 1, + "estimate": { + "a": 0.8979591836734694, + "b": 0.9069767441860465, + "ci": [ + -0.02020202020202022, + 0.05081826012058566 + ], + "diff": 0.009017560512577072, + "groups": 56, + "metric": "advisory_precision", + "resamples": 10000, + "se": 0.018550028052620342, + "units": 160 + }, + "mcnemar": null, + "target": null + }, + "advisory_recall": { + "ci_vs_zero": "spans", + "direction": 1, + "estimate": { + "a": 0.5, + "b": 0.4431818181818182, + "ci": [ + -0.1235955056179775, + 0.01010101010101011 + ], + "diff": -0.05681818181818182, + "groups": 56, + "metric": "advisory_recall", + "resamples": 10000, + "se": 0.033343898686966876, + "units": 160 + }, + "mcnemar": { + "b": 7, + "c": 2, + "concordant": 79, + "p_value": 0.1796875 + }, + "target": null + }, + "auc": { + "ci_vs_zero": "against", + "direction": 1, + "estimate": { + "a": 0.8012941919191919, + "b": 0.7802241161616161, + "ci": [ + -0.04139254385964919, + -0.0034279184917158645 + ], + "diff": -0.0210700757575758, + "groups": 56, + "metric": "auc", + "resamples": 10000, + "se": 0.009565373841115053, + "units": 160 + }, + "mcnemar": null, + "target": null + }, + "canary_agreement": { + "ci_vs_zero": "spans", + "direction": 1, + "estimate": { + "a": 0.875, + "b": 0.90625, + "ci": [ + -0.03448275862068961, + 0.0972972972972973 + ], + "diff": 0.03125, + "groups": 56, + "metric": "canary_agreement", + "resamples": 10000, + "se": 0.0337315848833629, + "units": 160 + }, + "mcnemar": { + "b": 10, + "c": 15, + "concordant": 135, + "p_value": 0.42435622215270996 + }, + "target": { + "b_meets": true, + "level": 0.9 + } + }, + "usd_per_candidate": { + "ci_vs_zero": "against", + "direction": -1, + "estimate": { + "a": 0.002131543750000001, + "b": 0.005102303125, + "ci": [ + 0.002724568181818181, + 0.00317872222222222 + ], + "diff": 0.002970759374999999, + "groups": 56, + "metric": "usd_per_candidate", + "resamples": 10000, + "se": 0.00011577410791736543, + "units": 160 + }, + "mcnemar": null, + "target": null + }, + "within_pair_auc": { + "ci_vs_zero": "against", + "direction": 1, + "estimate": { + "a": 0.8148148148148148, + "b": 0.7407407407407407, + "ci": [ + -0.12857142857142867, + -0.015151515151515138 + ], + "diff": -0.07407407407407407, + "groups": 56, + "metric": "within_pair_auc", + "resamples": 10000, + "se": 0.029550669251750233, + "units": 160 + }, + "mcnemar": null, + "target": null + } + }, + "tost": null, + "units": { + "by_label": { + "0": 72, + "1": 88 + }, + "groups": 56, + "outcomes": 640, + "paired": 160 + } +} diff --git a/benchmarks/measurements/2026-10-02-exp-002/record.json b/benchmarks/measurements/2026-10-02-exp-002/record.json new file mode 100644 index 0000000..dcdcafa --- /dev/null +++ b/benchmarks/measurements/2026-10-02-exp-002/record.json @@ -0,0 +1,377 @@ +{ + "created": "2026-10-02", + "decision": "reject", + "instruments": { + "distinct_scores_by_arm": { + "A": 72, + "B": 57 + }, + "outcomes_by_arm_replicate": { + "A:r0": 160, + "A:r1": 160, + "B:r0": 160, + "B:r1": 160 + }, + "parse_failed_by_arm": { + "A": 0, + "B": 0 + } + }, + "looks": [ + { + "boundary": 2.797, + "crossed": false, + "estimate": { + "a": 0.8611111111111112, + "b": 0.8888888888888888, + "ci": [ + -0.08333333333333326, + 0.1325301204819277 + ], + "diff": 0.02777777777777768, + "groups": 28, + "metric": "canary_agreement", + "resamples": 10000, + "se": 0.05449719877526111, + "units": 72 + }, + "fraction": 0.5, + "groups": 28, + "z": 0.5097101943226363 + }, + { + "boundary": 1.977, + "crossed": false, + "estimate": { + "a": 0.875, + "b": 0.90625, + "ci": [ + -0.03448275862068961, + 0.0972972972972973 + ], + "diff": 0.03125, + "groups": 56, + "metric": "canary_agreement", + "resamples": 10000, + "se": 0.0337315848833629, + "units": 160 + }, + "fraction": 1.0, + "groups": 56, + "z": 0.9264314175588332 + } + ], + "note": "COUNTS ONLY. Experiment exp-002 (learned-decision-engine task 7.4): arm A = the slice-2 compiled programs as they are (k = 5, sample 1 at temperature 0, samples 2..5 at 0.7, one retrieved example set); arm B = the same programs with ProgramSpec.sampling = retrieval_ensemble (k = 5 at temperature 0, each sample over a disjoint retrieved example set). Paired on the frozen units (every unit in both arms, two replicates per arm under different response-cache salts; replicate 1 is the canary), seeded repository order and a seeded per-unit coin for which arm goes first, the incumbent folds' seed on the 937-example static memory, 788 embedding vectors. Intervals: repository-cluster paired bootstrap 95% (10,000 resamples); exact McNemar on discordant units for binary metrics; two looks (O'Brien-Fleming 2.797 / 1.977). The overall table is the runner's result.json; the per-family table here uses the same estimator on each family's pairs. Driver: `ousast learn experiment run|analyse` (learn/experiments.py). No candidate, path, function or repository name.", + "overall": { + "advisory_precision": { + "A": 0.8979591836734694, + "B": 0.9069767441860465, + "ci95": [ + -0.02020202020202022, + 0.05081826012058566 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": 0.009017560512577072, + "mcnemar": null, + "target": null + }, + "advisory_recall": { + "A": 0.5, + "B": 0.4431818181818182, + "ci95": [ + -0.1235955056179775, + 0.01010101010101011 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": -0.05681818181818182, + "mcnemar": { + "b": 7, + "c": 2, + "concordant": 79, + "p_value": 0.1796875 + }, + "target": null + }, + "auc": { + "A": 0.8012941919191919, + "B": 0.7802241161616161, + "ci95": [ + -0.04139254385964919, + -0.0034279184917158645 + ], + "ci_vs_zero": "against", + "diff_B_minus_A": -0.0210700757575758, + "mcnemar": null, + "target": null + }, + "canary_agreement": { + "A": 0.875, + "B": 0.90625, + "ci95": [ + -0.03448275862068961, + 0.0972972972972973 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": 0.03125, + "mcnemar": { + "b": 10, + "c": 15, + "concordant": 135, + "p_value": 0.42435622215270996 + }, + "target": { + "b_meets": true, + "level": 0.9 + } + }, + "usd_per_candidate": { + "A": 0.002131543750000001, + "B": 0.005102303125, + "ci95": [ + 0.002724568181818181, + 0.00317872222222222 + ], + "ci_vs_zero": "against", + "diff_B_minus_A": 0.002970759374999999, + "mcnemar": null, + "target": null + }, + "within_pair_auc": { + "A": 0.8148148148148148, + "B": 0.7407407407407407, + "ci95": [ + -0.12857142857142867, + -0.015151515151515138 + ], + "ci_vs_zero": "against", + "diff_B_minus_A": -0.07407407407407407, + "mcnemar": null, + "target": null + } + }, + "per_family": { + "access_control": { + "groups": 24, + "metrics": { + "advisory_precision": { + "A": 1.0, + "B": 1.0, + "ci95": [ + 0.0, + 0.0 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": 0.0, + "mcnemar": null + }, + "advisory_recall": { + "A": 0.72, + "B": 0.68, + "ci95": [ + -0.16666666666666674, + 0.08333333333333337 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": -0.039999999999999925, + "mcnemar": { + "b": 2, + "c": 1, + "concordant": 22, + "p_value": 1.0 + } + }, + "auc": { + "A": 0.94625, + "B": 0.95125, + "ci95": [ + -0.023157894736842155, + 0.02941176470588236 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": 0.0050000000000000044, + "mcnemar": null + }, + "canary_agreement": { + "A": 0.926829268292683, + "B": 0.8780487804878049, + "ci95": [ + -0.18421052631578938, + 0.08108108108108103 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": -0.04878048780487809, + "mcnemar": { + "b": 5, + "c": 3, + "concordant": 33, + "p_value": 0.7265625 + } + }, + "usd_per_candidate": { + "A": 0.0019149512195121951, + "B": 0.0043439878048780484, + "ci95": [ + 0.0021030595238095253, + 0.00275970731707317 + ], + "ci_vs_zero": "against", + "diff_B_minus_A": 0.0024290365853658533, + "mcnemar": null + }, + "within_pair_auc": { + "A": 0.9375, + "B": 0.875, + "ci95": [ + -0.125, + 0.0 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": -0.0625, + "mcnemar": null + } + }, + "paired_units": 41 + }, + "injection": { + "groups": 34, + "metrics": { + "advisory_precision": { + "A": 0.8387096774193549, + "B": 0.8461538461538461, + "ci95": [ + -0.043269230769230727, + 0.06763285024154597 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": 0.007444168734491274, + "mcnemar": null + }, + "advisory_recall": { + "A": 0.4126984126984127, + "B": 0.3492063492063492, + "ci95": [ + -0.140625, + 0.0 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": -0.06349206349206349, + "mcnemar": { + "b": 5, + "c": 1, + "concordant": 57, + "p_value": 0.21875 + } + }, + "auc": { + "A": 0.7475907029478458, + "B": 0.7140022675736961, + "ci95": [ + -0.061038961038961004, + -0.011363636363636465 + ], + "ci_vs_zero": "against", + "diff_B_minus_A": -0.03358843537414968, + "mcnemar": null + }, + "canary_agreement": { + "A": 0.8571428571428571, + "B": 0.9159663865546218, + "ci95": [ + -0.016129032258064502, + 0.1322314049586777 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": 0.05882352941176472, + "mcnemar": { + "b": 5, + "c": 12, + "concordant": 102, + "p_value": 0.143463134765625 + } + }, + "usd_per_candidate": { + "A": 0.002206168067226891, + "B": 0.005363571428571428, + "ci95": [ + 0.0028887247706422007, + 0.0033776267605633826 + ], + "ci_vs_zero": "against", + "diff_B_minus_A": 0.003157403361344537, + "mcnemar": null + }, + "within_pair_auc": { + "A": 0.7631578947368421, + "B": 0.6842105263157895, + "ci95": [ + -0.15384615384615385, + 0.0 + ], + "ci_vs_zero": "spans", + "diff_B_minus_A": -0.07894736842105265, + "mcnemar": null + } + }, + "paired_units": 119 + } + }, + "reason": "the CI of within_pair_auc excludes 0 against B", + "registration": "benchmarks/experiments/exp-002-retrieval-ensemble/registration.json", + "result": "benchmarks/experiments/exp-002-retrieval-ensemble/result.json", + "run": { + "decided": { + "A": 320, + "B": 320 + }, + "folds_digest": "a1f78e141a572a4adcc9cd57eb8a784c440c177f0b60d5ef5bed9e94c8753c5f", + "reason": "", + "skipped": { + "A": 0, + "B": 0 + }, + "status": "done", + "stopped_at_look": null, + "units": 160 + }, + "spend": { + "caller": { + "A": { + "cache_digest": "39ba875897ccbc9dee39b07029d26e1fee84392d230776b63bb6f8aa2189d7b9", + "client_calls": 1206, + "replayed": 395, + "usd": 0.682094 + }, + "B": { + "cache_digest": "5f6187e9445f180fe234b2f48a984eb8fbecf0000552a4acc0a412780c6e628a", + "client_calls": 1350, + "replayed": 250, + "usd": 1.632737 + } + }, + "deepseek_balance_usd": { + "after": 16.04, + "before": 16.72, + "metered_delta": 0.68 + }, + "note": "priced usage prices every response a decision used, a replayed one the same as a paid one (Caller.usd); client_calls are the paid calls and replayed the response-cache hits (arm A: 33 units whose slice-2 sample-0 request was still cached, plus identical prompts across identical-twin candidates and retries; arm B: twins and retries only). The provider balance is read at the run's end and lags; the metered spend is bounded by the arm meters (A $1, B $3).", + "priced_usage_usd_by_arm": { + "A": 0.6821, + "B": 1.6327 + }, + "priced_usage_usd_by_arm_replicate": { + "A:r0": 0.4299, + "A:r1": 0.2522, + "B:r0": 1.3797, + "B:r1": 0.253 + } + }, + "units": { + "by_label": { + "0": 72, + "1": 88 + }, + "groups": 56, + "outcomes": 640, + "paired": 160 + } +}