mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
bench(learn): exp-002 result -- retrieval ensemble vs temperature sampling
Reject B. 160 paired units (injection 119, access_control 41; 56 repositories), both arms, two replicates each, no early stop. Canary agreement A 0.875 -> B 0.906 (CI [-0.03, +0.10], spans; B meets 0.90). Within-pair AUC A 0.815 -> B 0.741 (CI [-0.13, -0.02], against B). AUC -0.02 (against), ADVISORY recall -0.06 and precision +0.01 (span), usd per candidate A $0.0021 -> B $0.0051 (against). Priced usage A $0.68 / B $1.63; balance $16.72 -> $16.04 at the run's end (lags). Counts only; task 7.4 marked done. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
780caaa558
commit
48510fd5c2
@@ -139,8 +139,18 @@ a model or embedding call.
|
|||||||
on them, so `ousast learn experiment units` could not freeze the candidates.
|
on them, so `ousast learn experiment units` could not freeze the candidates.
|
||||||
- [ ] 7.3 exp-001 run and recorded (~$7.7 expected, ceiling $15) -- an inconclusive result is pre-registered as
|
- [ ] 7.3 exp-001 run and recorded (~$7.7 expected, ceiling $15) -- an inconclusive result is pre-registered as
|
||||||
likely; needs the frozen units file first.
|
likely; needs the frozen units file first.
|
||||||
- [ ] 7.4 exp-002 run and recorded **(budget go-ahead: ~$2 expected, ceiling $4)**; further program variants
|
- [x] 7.4 exp-002 run and recorded 2026-10-02 (budget go-ahead ~$2 expected, ceiling $4): **reject B**. 160 paired
|
||||||
(`k = 5` vs `k = 1`, the confidence source) as new manifests.
|
units (injection 119, access_control 41; 56 repositories; 108 units in 54 candidate pairs), both arms, two
|
||||||
|
replicates each, no early stop. Primary: canary agreement A 0.875 -> B 0.906 (diff +0.03, CI [-0.03, +0.10],
|
||||||
|
spans; B meets the 0.90 target), within-pair AUC A 0.815 -> B 0.741 (diff -0.07, CI [-0.13, -0.02], **against B**).
|
||||||
|
Secondary: AUC -0.02 (CI [-0.04, -0.003], against), ADVISORY recall -0.06 (spans; McNemar 7/2, p = 0.18),
|
||||||
|
precision +0.01 (spans), usd per candidate A $0.0021 -> B $0.0051 (CI [+0.0027, +0.0032], against: the five
|
||||||
|
disjoint example sets share only the prompt prefix in DeepSeek's cache). Priced usage A $0.68 (1206 paid calls,
|
||||||
|
395 replayed), B $1.63 (1350 paid, 250 replayed). Result: `benchmarks/experiments/exp-002-retrieval-ensemble/result.json`,
|
||||||
|
`benchmarks/measurements/2026-10-02-exp-002/record.json`. Three repairs on the way (each its own commit): example
|
||||||
|
rows' null plane-only features (1f525ff), the 788 embedding vectors absent from RustFS (copied from the file
|
||||||
|
store), the result row's column-type collision under S3 Select (780caaa). Further program variants (`k = 5` vs
|
||||||
|
`k = 1`, the confidence source) as new manifests.
|
||||||
- _Requirements: 3.6, 4.1-4.4_
|
- _Requirements: 3.6, 4.1-4.4_
|
||||||
|
|
||||||
- [ ] 8. Integration
|
- [ ] 8. Integration
|
||||||
|
|||||||
@@ -19,5 +19,5 @@
|
|||||||
"note": "frozen on the 937-example static memory (slice 2 evaluated 110 + 37 candidates on the 904-example memory of the same day; the recovered negatives of ef6945e moved the seeded compile split and the outer folds). Freezing needed two repairs first: example rows carried the plane-only features as null in x and the renderer refused them (1f525ff), and the RustFS store held none of the 788 embedding vectors the file store had (copied over with put_blob, no embedding call)."
|
"note": "frozen on the 937-example static memory (slice 2 evaluated 110 + 37 candidates on the 904-example memory of the same day; the recovered negatives of ef6945e moved the seeded compile split and the outer folds). Freezing needed two repairs first: example rows carried the plane-only features as null in x and the renderer refused them (1f525ff), and the RustFS store held none of the 788 embedding vectors the file store had (copied over with put_blob, no embedding call)."
|
||||||
},
|
},
|
||||||
"first_registration": {"registered": "2026-10-02", "pin": "fbc3220bbe1c8a4f8afa0a724fd41a51b390dd22", "units_status": "pending"},
|
"first_registration": {"registered": "2026-10-02", "pin": "fbc3220bbe1c8a4f8afa0a724fd41a51b390dd22", "units_status": "pending"},
|
||||||
"status": "registered, units frozen; run pending"
|
"status": "run and analysed 2026-10-02: reject (result.json; benchmarks/measurements/2026-10-02-exp-002/record.json)"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,241 @@
|
|||||||
|
{
|
||||||
|
"alpha": 0.05,
|
||||||
|
"arms": {
|
||||||
|
"A": {
|
||||||
|
"budget_usd": 1.0,
|
||||||
|
"label": "today's sampling -- k = 5, sample 1 at temperature 0 and samples 2..5 at 0.7, one retrieved example set",
|
||||||
|
"repeats": 2,
|
||||||
|
"replay_only": false
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"budget_usd": 3.0,
|
||||||
|
"label": "retrieval ensemble -- k = 5 at temperature 0, each sample over a disjoint retrieved example set",
|
||||||
|
"repeats": 2,
|
||||||
|
"replay_only": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"code_commit": "780caaa558f2cc091fd6b0fd5c618c3b488030c2",
|
||||||
|
"decision": "reject",
|
||||||
|
"experiment": "exp-002-retrieval-ensemble",
|
||||||
|
"hypothesis": "Drawing the k = 5 samples from five disjoint retrieved example sets at temperature 0 (a retrieval ensemble) makes the decision more repeatable than five temperature samples over one example set, without hurting the ranking within a candidate pair, at no higher cost per candidate.\n",
|
||||||
|
"looks": [
|
||||||
|
{
|
||||||
|
"boundary": 2.797,
|
||||||
|
"crossed": false,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.8611111111111112,
|
||||||
|
"b": 0.8888888888888888,
|
||||||
|
"ci": [
|
||||||
|
-0.08333333333333326,
|
||||||
|
0.1325301204819277
|
||||||
|
],
|
||||||
|
"diff": 0.02777777777777768,
|
||||||
|
"groups": 28,
|
||||||
|
"metric": "canary_agreement",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.05449719877526111,
|
||||||
|
"units": 72
|
||||||
|
},
|
||||||
|
"fraction": 0.5,
|
||||||
|
"groups": 28,
|
||||||
|
"z": 0.5097101943226363
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"boundary": 1.977,
|
||||||
|
"crossed": false,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.875,
|
||||||
|
"b": 0.90625,
|
||||||
|
"ci": [
|
||||||
|
-0.03448275862068961,
|
||||||
|
0.0972972972972973
|
||||||
|
],
|
||||||
|
"diff": 0.03125,
|
||||||
|
"groups": 56,
|
||||||
|
"metric": "canary_agreement",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.0337315848833629,
|
||||||
|
"units": 160
|
||||||
|
},
|
||||||
|
"fraction": 1.0,
|
||||||
|
"groups": 56,
|
||||||
|
"z": 0.9264314175588332
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"manifest_digest": "f5004fb8618b1f7cc7dcf298d42b4ac1b6c0c1aa72a97299c77f937bd6175148",
|
||||||
|
"mde": 0.1,
|
||||||
|
"primary": [
|
||||||
|
{
|
||||||
|
"name": "canary_agreement",
|
||||||
|
"target": 0.9
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "within_pair_auc"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"reason": "the CI of within_pair_auc excludes 0 against B",
|
||||||
|
"resamples": 10000,
|
||||||
|
"secondary": [
|
||||||
|
{
|
||||||
|
"name": "usd_per_candidate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "auc"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "advisory_recall"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "advisory_precision"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"spend_usd": {
|
||||||
|
"A": 0.682094,
|
||||||
|
"B": 1.632737
|
||||||
|
},
|
||||||
|
"tests": {
|
||||||
|
"advisory_precision": {
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"direction": 1,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.8979591836734694,
|
||||||
|
"b": 0.9069767441860465,
|
||||||
|
"ci": [
|
||||||
|
-0.02020202020202022,
|
||||||
|
0.05081826012058566
|
||||||
|
],
|
||||||
|
"diff": 0.009017560512577072,
|
||||||
|
"groups": 56,
|
||||||
|
"metric": "advisory_precision",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.018550028052620342,
|
||||||
|
"units": 160
|
||||||
|
},
|
||||||
|
"mcnemar": null,
|
||||||
|
"target": null
|
||||||
|
},
|
||||||
|
"advisory_recall": {
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"direction": 1,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.5,
|
||||||
|
"b": 0.4431818181818182,
|
||||||
|
"ci": [
|
||||||
|
-0.1235955056179775,
|
||||||
|
0.01010101010101011
|
||||||
|
],
|
||||||
|
"diff": -0.05681818181818182,
|
||||||
|
"groups": 56,
|
||||||
|
"metric": "advisory_recall",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.033343898686966876,
|
||||||
|
"units": 160
|
||||||
|
},
|
||||||
|
"mcnemar": {
|
||||||
|
"b": 7,
|
||||||
|
"c": 2,
|
||||||
|
"concordant": 79,
|
||||||
|
"p_value": 0.1796875
|
||||||
|
},
|
||||||
|
"target": null
|
||||||
|
},
|
||||||
|
"auc": {
|
||||||
|
"ci_vs_zero": "against",
|
||||||
|
"direction": 1,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.8012941919191919,
|
||||||
|
"b": 0.7802241161616161,
|
||||||
|
"ci": [
|
||||||
|
-0.04139254385964919,
|
||||||
|
-0.0034279184917158645
|
||||||
|
],
|
||||||
|
"diff": -0.0210700757575758,
|
||||||
|
"groups": 56,
|
||||||
|
"metric": "auc",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.009565373841115053,
|
||||||
|
"units": 160
|
||||||
|
},
|
||||||
|
"mcnemar": null,
|
||||||
|
"target": null
|
||||||
|
},
|
||||||
|
"canary_agreement": {
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"direction": 1,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.875,
|
||||||
|
"b": 0.90625,
|
||||||
|
"ci": [
|
||||||
|
-0.03448275862068961,
|
||||||
|
0.0972972972972973
|
||||||
|
],
|
||||||
|
"diff": 0.03125,
|
||||||
|
"groups": 56,
|
||||||
|
"metric": "canary_agreement",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.0337315848833629,
|
||||||
|
"units": 160
|
||||||
|
},
|
||||||
|
"mcnemar": {
|
||||||
|
"b": 10,
|
||||||
|
"c": 15,
|
||||||
|
"concordant": 135,
|
||||||
|
"p_value": 0.42435622215270996
|
||||||
|
},
|
||||||
|
"target": {
|
||||||
|
"b_meets": true,
|
||||||
|
"level": 0.9
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"usd_per_candidate": {
|
||||||
|
"ci_vs_zero": "against",
|
||||||
|
"direction": -1,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.002131543750000001,
|
||||||
|
"b": 0.005102303125,
|
||||||
|
"ci": [
|
||||||
|
0.002724568181818181,
|
||||||
|
0.00317872222222222
|
||||||
|
],
|
||||||
|
"diff": 0.002970759374999999,
|
||||||
|
"groups": 56,
|
||||||
|
"metric": "usd_per_candidate",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.00011577410791736543,
|
||||||
|
"units": 160
|
||||||
|
},
|
||||||
|
"mcnemar": null,
|
||||||
|
"target": null
|
||||||
|
},
|
||||||
|
"within_pair_auc": {
|
||||||
|
"ci_vs_zero": "against",
|
||||||
|
"direction": 1,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.8148148148148148,
|
||||||
|
"b": 0.7407407407407407,
|
||||||
|
"ci": [
|
||||||
|
-0.12857142857142867,
|
||||||
|
-0.015151515151515138
|
||||||
|
],
|
||||||
|
"diff": -0.07407407407407407,
|
||||||
|
"groups": 56,
|
||||||
|
"metric": "within_pair_auc",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.029550669251750233,
|
||||||
|
"units": 160
|
||||||
|
},
|
||||||
|
"mcnemar": null,
|
||||||
|
"target": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tost": null,
|
||||||
|
"units": {
|
||||||
|
"by_label": {
|
||||||
|
"0": 72,
|
||||||
|
"1": 88
|
||||||
|
},
|
||||||
|
"groups": 56,
|
||||||
|
"outcomes": 640,
|
||||||
|
"paired": 160
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,377 @@
|
|||||||
|
{
|
||||||
|
"created": "2026-10-02",
|
||||||
|
"decision": "reject",
|
||||||
|
"instruments": {
|
||||||
|
"distinct_scores_by_arm": {
|
||||||
|
"A": 72,
|
||||||
|
"B": 57
|
||||||
|
},
|
||||||
|
"outcomes_by_arm_replicate": {
|
||||||
|
"A:r0": 160,
|
||||||
|
"A:r1": 160,
|
||||||
|
"B:r0": 160,
|
||||||
|
"B:r1": 160
|
||||||
|
},
|
||||||
|
"parse_failed_by_arm": {
|
||||||
|
"A": 0,
|
||||||
|
"B": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"looks": [
|
||||||
|
{
|
||||||
|
"boundary": 2.797,
|
||||||
|
"crossed": false,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.8611111111111112,
|
||||||
|
"b": 0.8888888888888888,
|
||||||
|
"ci": [
|
||||||
|
-0.08333333333333326,
|
||||||
|
0.1325301204819277
|
||||||
|
],
|
||||||
|
"diff": 0.02777777777777768,
|
||||||
|
"groups": 28,
|
||||||
|
"metric": "canary_agreement",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.05449719877526111,
|
||||||
|
"units": 72
|
||||||
|
},
|
||||||
|
"fraction": 0.5,
|
||||||
|
"groups": 28,
|
||||||
|
"z": 0.5097101943226363
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"boundary": 1.977,
|
||||||
|
"crossed": false,
|
||||||
|
"estimate": {
|
||||||
|
"a": 0.875,
|
||||||
|
"b": 0.90625,
|
||||||
|
"ci": [
|
||||||
|
-0.03448275862068961,
|
||||||
|
0.0972972972972973
|
||||||
|
],
|
||||||
|
"diff": 0.03125,
|
||||||
|
"groups": 56,
|
||||||
|
"metric": "canary_agreement",
|
||||||
|
"resamples": 10000,
|
||||||
|
"se": 0.0337315848833629,
|
||||||
|
"units": 160
|
||||||
|
},
|
||||||
|
"fraction": 1.0,
|
||||||
|
"groups": 56,
|
||||||
|
"z": 0.9264314175588332
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"note": "COUNTS ONLY. Experiment exp-002 (learned-decision-engine task 7.4): arm A = the slice-2 compiled programs as they are (k = 5, sample 1 at temperature 0, samples 2..5 at 0.7, one retrieved example set); arm B = the same programs with ProgramSpec.sampling = retrieval_ensemble (k = 5 at temperature 0, each sample over a disjoint retrieved example set). Paired on the frozen units (every unit in both arms, two replicates per arm under different response-cache salts; replicate 1 is the canary), seeded repository order and a seeded per-unit coin for which arm goes first, the incumbent folds' seed on the 937-example static memory, 788 embedding vectors. Intervals: repository-cluster paired bootstrap 95% (10,000 resamples); exact McNemar on discordant units for binary metrics; two looks (O'Brien-Fleming 2.797 / 1.977). The overall table is the runner's result.json; the per-family table here uses the same estimator on each family's pairs. Driver: `ousast learn experiment run|analyse` (learn/experiments.py). No candidate, path, function or repository name.",
|
||||||
|
"overall": {
|
||||||
|
"advisory_precision": {
|
||||||
|
"A": 0.8979591836734694,
|
||||||
|
"B": 0.9069767441860465,
|
||||||
|
"ci95": [
|
||||||
|
-0.02020202020202022,
|
||||||
|
0.05081826012058566
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": 0.009017560512577072,
|
||||||
|
"mcnemar": null,
|
||||||
|
"target": null
|
||||||
|
},
|
||||||
|
"advisory_recall": {
|
||||||
|
"A": 0.5,
|
||||||
|
"B": 0.4431818181818182,
|
||||||
|
"ci95": [
|
||||||
|
-0.1235955056179775,
|
||||||
|
0.01010101010101011
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": -0.05681818181818182,
|
||||||
|
"mcnemar": {
|
||||||
|
"b": 7,
|
||||||
|
"c": 2,
|
||||||
|
"concordant": 79,
|
||||||
|
"p_value": 0.1796875
|
||||||
|
},
|
||||||
|
"target": null
|
||||||
|
},
|
||||||
|
"auc": {
|
||||||
|
"A": 0.8012941919191919,
|
||||||
|
"B": 0.7802241161616161,
|
||||||
|
"ci95": [
|
||||||
|
-0.04139254385964919,
|
||||||
|
-0.0034279184917158645
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "against",
|
||||||
|
"diff_B_minus_A": -0.0210700757575758,
|
||||||
|
"mcnemar": null,
|
||||||
|
"target": null
|
||||||
|
},
|
||||||
|
"canary_agreement": {
|
||||||
|
"A": 0.875,
|
||||||
|
"B": 0.90625,
|
||||||
|
"ci95": [
|
||||||
|
-0.03448275862068961,
|
||||||
|
0.0972972972972973
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": 0.03125,
|
||||||
|
"mcnemar": {
|
||||||
|
"b": 10,
|
||||||
|
"c": 15,
|
||||||
|
"concordant": 135,
|
||||||
|
"p_value": 0.42435622215270996
|
||||||
|
},
|
||||||
|
"target": {
|
||||||
|
"b_meets": true,
|
||||||
|
"level": 0.9
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"usd_per_candidate": {
|
||||||
|
"A": 0.002131543750000001,
|
||||||
|
"B": 0.005102303125,
|
||||||
|
"ci95": [
|
||||||
|
0.002724568181818181,
|
||||||
|
0.00317872222222222
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "against",
|
||||||
|
"diff_B_minus_A": 0.002970759374999999,
|
||||||
|
"mcnemar": null,
|
||||||
|
"target": null
|
||||||
|
},
|
||||||
|
"within_pair_auc": {
|
||||||
|
"A": 0.8148148148148148,
|
||||||
|
"B": 0.7407407407407407,
|
||||||
|
"ci95": [
|
||||||
|
-0.12857142857142867,
|
||||||
|
-0.015151515151515138
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "against",
|
||||||
|
"diff_B_minus_A": -0.07407407407407407,
|
||||||
|
"mcnemar": null,
|
||||||
|
"target": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"per_family": {
|
||||||
|
"access_control": {
|
||||||
|
"groups": 24,
|
||||||
|
"metrics": {
|
||||||
|
"advisory_precision": {
|
||||||
|
"A": 1.0,
|
||||||
|
"B": 1.0,
|
||||||
|
"ci95": [
|
||||||
|
0.0,
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": 0.0,
|
||||||
|
"mcnemar": null
|
||||||
|
},
|
||||||
|
"advisory_recall": {
|
||||||
|
"A": 0.72,
|
||||||
|
"B": 0.68,
|
||||||
|
"ci95": [
|
||||||
|
-0.16666666666666674,
|
||||||
|
0.08333333333333337
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": -0.039999999999999925,
|
||||||
|
"mcnemar": {
|
||||||
|
"b": 2,
|
||||||
|
"c": 1,
|
||||||
|
"concordant": 22,
|
||||||
|
"p_value": 1.0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"auc": {
|
||||||
|
"A": 0.94625,
|
||||||
|
"B": 0.95125,
|
||||||
|
"ci95": [
|
||||||
|
-0.023157894736842155,
|
||||||
|
0.02941176470588236
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": 0.0050000000000000044,
|
||||||
|
"mcnemar": null
|
||||||
|
},
|
||||||
|
"canary_agreement": {
|
||||||
|
"A": 0.926829268292683,
|
||||||
|
"B": 0.8780487804878049,
|
||||||
|
"ci95": [
|
||||||
|
-0.18421052631578938,
|
||||||
|
0.08108108108108103
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": -0.04878048780487809,
|
||||||
|
"mcnemar": {
|
||||||
|
"b": 5,
|
||||||
|
"c": 3,
|
||||||
|
"concordant": 33,
|
||||||
|
"p_value": 0.7265625
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"usd_per_candidate": {
|
||||||
|
"A": 0.0019149512195121951,
|
||||||
|
"B": 0.0043439878048780484,
|
||||||
|
"ci95": [
|
||||||
|
0.0021030595238095253,
|
||||||
|
0.00275970731707317
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "against",
|
||||||
|
"diff_B_minus_A": 0.0024290365853658533,
|
||||||
|
"mcnemar": null
|
||||||
|
},
|
||||||
|
"within_pair_auc": {
|
||||||
|
"A": 0.9375,
|
||||||
|
"B": 0.875,
|
||||||
|
"ci95": [
|
||||||
|
-0.125,
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": -0.0625,
|
||||||
|
"mcnemar": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"paired_units": 41
|
||||||
|
},
|
||||||
|
"injection": {
|
||||||
|
"groups": 34,
|
||||||
|
"metrics": {
|
||||||
|
"advisory_precision": {
|
||||||
|
"A": 0.8387096774193549,
|
||||||
|
"B": 0.8461538461538461,
|
||||||
|
"ci95": [
|
||||||
|
-0.043269230769230727,
|
||||||
|
0.06763285024154597
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": 0.007444168734491274,
|
||||||
|
"mcnemar": null
|
||||||
|
},
|
||||||
|
"advisory_recall": {
|
||||||
|
"A": 0.4126984126984127,
|
||||||
|
"B": 0.3492063492063492,
|
||||||
|
"ci95": [
|
||||||
|
-0.140625,
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": -0.06349206349206349,
|
||||||
|
"mcnemar": {
|
||||||
|
"b": 5,
|
||||||
|
"c": 1,
|
||||||
|
"concordant": 57,
|
||||||
|
"p_value": 0.21875
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"auc": {
|
||||||
|
"A": 0.7475907029478458,
|
||||||
|
"B": 0.7140022675736961,
|
||||||
|
"ci95": [
|
||||||
|
-0.061038961038961004,
|
||||||
|
-0.011363636363636465
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "against",
|
||||||
|
"diff_B_minus_A": -0.03358843537414968,
|
||||||
|
"mcnemar": null
|
||||||
|
},
|
||||||
|
"canary_agreement": {
|
||||||
|
"A": 0.8571428571428571,
|
||||||
|
"B": 0.9159663865546218,
|
||||||
|
"ci95": [
|
||||||
|
-0.016129032258064502,
|
||||||
|
0.1322314049586777
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": 0.05882352941176472,
|
||||||
|
"mcnemar": {
|
||||||
|
"b": 5,
|
||||||
|
"c": 12,
|
||||||
|
"concordant": 102,
|
||||||
|
"p_value": 0.143463134765625
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"usd_per_candidate": {
|
||||||
|
"A": 0.002206168067226891,
|
||||||
|
"B": 0.005363571428571428,
|
||||||
|
"ci95": [
|
||||||
|
0.0028887247706422007,
|
||||||
|
0.0033776267605633826
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "against",
|
||||||
|
"diff_B_minus_A": 0.003157403361344537,
|
||||||
|
"mcnemar": null
|
||||||
|
},
|
||||||
|
"within_pair_auc": {
|
||||||
|
"A": 0.7631578947368421,
|
||||||
|
"B": 0.6842105263157895,
|
||||||
|
"ci95": [
|
||||||
|
-0.15384615384615385,
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
"ci_vs_zero": "spans",
|
||||||
|
"diff_B_minus_A": -0.07894736842105265,
|
||||||
|
"mcnemar": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"paired_units": 119
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"reason": "the CI of within_pair_auc excludes 0 against B",
|
||||||
|
"registration": "benchmarks/experiments/exp-002-retrieval-ensemble/registration.json",
|
||||||
|
"result": "benchmarks/experiments/exp-002-retrieval-ensemble/result.json",
|
||||||
|
"run": {
|
||||||
|
"decided": {
|
||||||
|
"A": 320,
|
||||||
|
"B": 320
|
||||||
|
},
|
||||||
|
"folds_digest": "a1f78e141a572a4adcc9cd57eb8a784c440c177f0b60d5ef5bed9e94c8753c5f",
|
||||||
|
"reason": "",
|
||||||
|
"skipped": {
|
||||||
|
"A": 0,
|
||||||
|
"B": 0
|
||||||
|
},
|
||||||
|
"status": "done",
|
||||||
|
"stopped_at_look": null,
|
||||||
|
"units": 160
|
||||||
|
},
|
||||||
|
"spend": {
|
||||||
|
"caller": {
|
||||||
|
"A": {
|
||||||
|
"cache_digest": "39ba875897ccbc9dee39b07029d26e1fee84392d230776b63bb6f8aa2189d7b9",
|
||||||
|
"client_calls": 1206,
|
||||||
|
"replayed": 395,
|
||||||
|
"usd": 0.682094
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
"cache_digest": "5f6187e9445f180fe234b2f48a984eb8fbecf0000552a4acc0a412780c6e628a",
|
||||||
|
"client_calls": 1350,
|
||||||
|
"replayed": 250,
|
||||||
|
"usd": 1.632737
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"deepseek_balance_usd": {
|
||||||
|
"after": 16.04,
|
||||||
|
"before": 16.72,
|
||||||
|
"metered_delta": 0.68
|
||||||
|
},
|
||||||
|
"note": "priced usage prices every response a decision used, a replayed one the same as a paid one (Caller.usd); client_calls are the paid calls and replayed the response-cache hits (arm A: 33 units whose slice-2 sample-0 request was still cached, plus identical prompts across identical-twin candidates and retries; arm B: twins and retries only). The provider balance is read at the run's end and lags; the metered spend is bounded by the arm meters (A $1, B $3).",
|
||||||
|
"priced_usage_usd_by_arm": {
|
||||||
|
"A": 0.6821,
|
||||||
|
"B": 1.6327
|
||||||
|
},
|
||||||
|
"priced_usage_usd_by_arm_replicate": {
|
||||||
|
"A:r0": 0.4299,
|
||||||
|
"A:r1": 0.2522,
|
||||||
|
"B:r0": 1.3797,
|
||||||
|
"B:r1": 0.253
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"units": {
|
||||||
|
"by_label": {
|
||||||
|
"0": 72,
|
||||||
|
"1": 88
|
||||||
|
},
|
||||||
|
"groups": 56,
|
||||||
|
"outcomes": 640,
|
||||||
|
"paired": 160
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user