mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 03:59:28 +00:00
bench(learn): exp-002 result -- retrieval ensemble vs temperature sampling
Reject B. 160 paired units (injection 119, access_control 41; 56 repositories), both arms, two replicates each, no early stop. Canary agreement A 0.875 -> B 0.906 (CI [-0.03, +0.10], spans; B meets 0.90). Within-pair AUC A 0.815 -> B 0.741 (CI [-0.13, -0.02], against B). AUC -0.02 (against), ADVISORY recall -0.06 and precision +0.01 (span), usd per candidate A $0.0021 -> B $0.0051 (against). Priced usage A $0.68 / B $1.63; balance $16.72 -> $16.04 at the run's end (lags). Counts only; task 7.4 marked done. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
780caaa558
commit
48510fd5c2
@@ -19,5 +19,5 @@
|
||||
"note": "frozen on the 937-example static memory (slice 2 evaluated 110 + 37 candidates on the 904-example memory of the same day; the recovered negatives of ef6945e moved the seeded compile split and the outer folds). Freezing needed two repairs first: example rows carried the plane-only features as null in x and the renderer refused them (1f525ff), and the RustFS store held none of the 788 embedding vectors the file store had (copied over with put_blob, no embedding call)."
|
||||
},
|
||||
"first_registration": {"registered": "2026-10-02", "pin": "fbc3220bbe1c8a4f8afa0a724fd41a51b390dd22", "units_status": "pending"},
|
||||
"status": "registered, units frozen; run pending"
|
||||
"status": "run and analysed 2026-10-02: reject (result.json; benchmarks/measurements/2026-10-02-exp-002/record.json)"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
{
|
||||
"alpha": 0.05,
|
||||
"arms": {
|
||||
"A": {
|
||||
"budget_usd": 1.0,
|
||||
"label": "today's sampling -- k = 5, sample 1 at temperature 0 and samples 2..5 at 0.7, one retrieved example set",
|
||||
"repeats": 2,
|
||||
"replay_only": false
|
||||
},
|
||||
"B": {
|
||||
"budget_usd": 3.0,
|
||||
"label": "retrieval ensemble -- k = 5 at temperature 0, each sample over a disjoint retrieved example set",
|
||||
"repeats": 2,
|
||||
"replay_only": false
|
||||
}
|
||||
},
|
||||
"code_commit": "780caaa558f2cc091fd6b0fd5c618c3b488030c2",
|
||||
"decision": "reject",
|
||||
"experiment": "exp-002-retrieval-ensemble",
|
||||
"hypothesis": "Drawing the k = 5 samples from five disjoint retrieved example sets at temperature 0 (a retrieval ensemble) makes the decision more repeatable than five temperature samples over one example set, without hurting the ranking within a candidate pair, at no higher cost per candidate.\n",
|
||||
"looks": [
|
||||
{
|
||||
"boundary": 2.797,
|
||||
"crossed": false,
|
||||
"estimate": {
|
||||
"a": 0.8611111111111112,
|
||||
"b": 0.8888888888888888,
|
||||
"ci": [
|
||||
-0.08333333333333326,
|
||||
0.1325301204819277
|
||||
],
|
||||
"diff": 0.02777777777777768,
|
||||
"groups": 28,
|
||||
"metric": "canary_agreement",
|
||||
"resamples": 10000,
|
||||
"se": 0.05449719877526111,
|
||||
"units": 72
|
||||
},
|
||||
"fraction": 0.5,
|
||||
"groups": 28,
|
||||
"z": 0.5097101943226363
|
||||
},
|
||||
{
|
||||
"boundary": 1.977,
|
||||
"crossed": false,
|
||||
"estimate": {
|
||||
"a": 0.875,
|
||||
"b": 0.90625,
|
||||
"ci": [
|
||||
-0.03448275862068961,
|
||||
0.0972972972972973
|
||||
],
|
||||
"diff": 0.03125,
|
||||
"groups": 56,
|
||||
"metric": "canary_agreement",
|
||||
"resamples": 10000,
|
||||
"se": 0.0337315848833629,
|
||||
"units": 160
|
||||
},
|
||||
"fraction": 1.0,
|
||||
"groups": 56,
|
||||
"z": 0.9264314175588332
|
||||
}
|
||||
],
|
||||
"manifest_digest": "f5004fb8618b1f7cc7dcf298d42b4ac1b6c0c1aa72a97299c77f937bd6175148",
|
||||
"mde": 0.1,
|
||||
"primary": [
|
||||
{
|
||||
"name": "canary_agreement",
|
||||
"target": 0.9
|
||||
},
|
||||
{
|
||||
"name": "within_pair_auc"
|
||||
}
|
||||
],
|
||||
"reason": "the CI of within_pair_auc excludes 0 against B",
|
||||
"resamples": 10000,
|
||||
"secondary": [
|
||||
{
|
||||
"name": "usd_per_candidate"
|
||||
},
|
||||
{
|
||||
"name": "auc"
|
||||
},
|
||||
{
|
||||
"name": "advisory_recall"
|
||||
},
|
||||
{
|
||||
"name": "advisory_precision"
|
||||
}
|
||||
],
|
||||
"spend_usd": {
|
||||
"A": 0.682094,
|
||||
"B": 1.632737
|
||||
},
|
||||
"tests": {
|
||||
"advisory_precision": {
|
||||
"ci_vs_zero": "spans",
|
||||
"direction": 1,
|
||||
"estimate": {
|
||||
"a": 0.8979591836734694,
|
||||
"b": 0.9069767441860465,
|
||||
"ci": [
|
||||
-0.02020202020202022,
|
||||
0.05081826012058566
|
||||
],
|
||||
"diff": 0.009017560512577072,
|
||||
"groups": 56,
|
||||
"metric": "advisory_precision",
|
||||
"resamples": 10000,
|
||||
"se": 0.018550028052620342,
|
||||
"units": 160
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
},
|
||||
"advisory_recall": {
|
||||
"ci_vs_zero": "spans",
|
||||
"direction": 1,
|
||||
"estimate": {
|
||||
"a": 0.5,
|
||||
"b": 0.4431818181818182,
|
||||
"ci": [
|
||||
-0.1235955056179775,
|
||||
0.01010101010101011
|
||||
],
|
||||
"diff": -0.05681818181818182,
|
||||
"groups": 56,
|
||||
"metric": "advisory_recall",
|
||||
"resamples": 10000,
|
||||
"se": 0.033343898686966876,
|
||||
"units": 160
|
||||
},
|
||||
"mcnemar": {
|
||||
"b": 7,
|
||||
"c": 2,
|
||||
"concordant": 79,
|
||||
"p_value": 0.1796875
|
||||
},
|
||||
"target": null
|
||||
},
|
||||
"auc": {
|
||||
"ci_vs_zero": "against",
|
||||
"direction": 1,
|
||||
"estimate": {
|
||||
"a": 0.8012941919191919,
|
||||
"b": 0.7802241161616161,
|
||||
"ci": [
|
||||
-0.04139254385964919,
|
||||
-0.0034279184917158645
|
||||
],
|
||||
"diff": -0.0210700757575758,
|
||||
"groups": 56,
|
||||
"metric": "auc",
|
||||
"resamples": 10000,
|
||||
"se": 0.009565373841115053,
|
||||
"units": 160
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
},
|
||||
"canary_agreement": {
|
||||
"ci_vs_zero": "spans",
|
||||
"direction": 1,
|
||||
"estimate": {
|
||||
"a": 0.875,
|
||||
"b": 0.90625,
|
||||
"ci": [
|
||||
-0.03448275862068961,
|
||||
0.0972972972972973
|
||||
],
|
||||
"diff": 0.03125,
|
||||
"groups": 56,
|
||||
"metric": "canary_agreement",
|
||||
"resamples": 10000,
|
||||
"se": 0.0337315848833629,
|
||||
"units": 160
|
||||
},
|
||||
"mcnemar": {
|
||||
"b": 10,
|
||||
"c": 15,
|
||||
"concordant": 135,
|
||||
"p_value": 0.42435622215270996
|
||||
},
|
||||
"target": {
|
||||
"b_meets": true,
|
||||
"level": 0.9
|
||||
}
|
||||
},
|
||||
"usd_per_candidate": {
|
||||
"ci_vs_zero": "against",
|
||||
"direction": -1,
|
||||
"estimate": {
|
||||
"a": 0.002131543750000001,
|
||||
"b": 0.005102303125,
|
||||
"ci": [
|
||||
0.002724568181818181,
|
||||
0.00317872222222222
|
||||
],
|
||||
"diff": 0.002970759374999999,
|
||||
"groups": 56,
|
||||
"metric": "usd_per_candidate",
|
||||
"resamples": 10000,
|
||||
"se": 0.00011577410791736543,
|
||||
"units": 160
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
},
|
||||
"within_pair_auc": {
|
||||
"ci_vs_zero": "against",
|
||||
"direction": 1,
|
||||
"estimate": {
|
||||
"a": 0.8148148148148148,
|
||||
"b": 0.7407407407407407,
|
||||
"ci": [
|
||||
-0.12857142857142867,
|
||||
-0.015151515151515138
|
||||
],
|
||||
"diff": -0.07407407407407407,
|
||||
"groups": 56,
|
||||
"metric": "within_pair_auc",
|
||||
"resamples": 10000,
|
||||
"se": 0.029550669251750233,
|
||||
"units": 160
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
}
|
||||
},
|
||||
"tost": null,
|
||||
"units": {
|
||||
"by_label": {
|
||||
"0": 72,
|
||||
"1": 88
|
||||
},
|
||||
"groups": 56,
|
||||
"outcomes": 640,
|
||||
"paired": 160
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user