mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
learn: exp-005 graph slice -- inconclusive (within-pair AUC 0.762 -> 0.746, CI [-0.063, 0.032]); unsure rate 0.29 -> 0.24; fill \$1.59
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
0acbc7ec41
commit
094f6d226c
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"arms": [
|
||||
"B"
|
||||
],
|
||||
"client_calls": 2394,
|
||||
"experiment": "exp-005-graph-slice",
|
||||
"families": {
|
||||
"access_control": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 330,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 0.9956892000000007,
|
||||
"filled": 330,
|
||||
"missing": 330,
|
||||
"requests": 330,
|
||||
"still_missing": 0,
|
||||
"units": 66
|
||||
},
|
||||
"deserialization": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 310,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 0.8843295999999988,
|
||||
"filled": 310,
|
||||
"missing": 310,
|
||||
"requests": 310,
|
||||
"still_missing": 0,
|
||||
"units": 64
|
||||
},
|
||||
"injection": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 675,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 2.270278999999999,
|
||||
"filled": 675,
|
||||
"missing": 675,
|
||||
"requests": 675,
|
||||
"still_missing": 0,
|
||||
"units": 149
|
||||
},
|
||||
"output_encoding": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 325,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 1.0014180000000004,
|
||||
"filled": 328,
|
||||
"missing": 325,
|
||||
"requests": 328,
|
||||
"still_missing": 0,
|
||||
"units": 69
|
||||
},
|
||||
"path": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 320,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 0.9399192000000005,
|
||||
"filled": 325,
|
||||
"missing": 320,
|
||||
"requests": 325,
|
||||
"still_missing": 0,
|
||||
"units": 66
|
||||
},
|
||||
"untrusted_destination": {
|
||||
"cached": 0,
|
||||
"conditional_retries": 425,
|
||||
"estimate_method": "estimated",
|
||||
"estimated_output_tokens": 300,
|
||||
"estimated_usd": 1.174316,
|
||||
"filled": 426,
|
||||
"missing": 425,
|
||||
"requests": 426,
|
||||
"still_missing": 0,
|
||||
"units": 87
|
||||
}
|
||||
},
|
||||
"reason": "",
|
||||
"status": "done",
|
||||
"usd": 1.5890382240000016
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
{
|
||||
"alpha": 0.05,
|
||||
"arms": {
|
||||
"A": {
|
||||
"budget_usd": 0.0,
|
||||
"label": "six slice-2 incumbents unchanged, replaying exp-004's response cache",
|
||||
"repeats": 1,
|
||||
"replay_only": true
|
||||
},
|
||||
"B": {
|
||||
"budget_usd": 6.0,
|
||||
"label": "the same six programs with the in-function slice added to their prompts",
|
||||
"repeats": 1,
|
||||
"replay_only": false
|
||||
}
|
||||
},
|
||||
"code_commit": "0acbc7ec41dcaeca805c4720bc3f6d02cb940ee7",
|
||||
"decision": "inconclusive",
|
||||
"experiment": "exp-005-graph-slice",
|
||||
"hypothesis": "Adding the in-function def-use slice to the prompt improves within-pair ranking of the six slice-2 incumbent programs on held-out repositories, at no more than 1.5 times the cost per candidate.\n",
|
||||
"looks": [
|
||||
{
|
||||
"boundary": 1.959964,
|
||||
"crossed": false,
|
||||
"estimate": {
|
||||
"a": 0.7615384615384615,
|
||||
"b": 0.7461538461538462,
|
||||
"ci": [
|
||||
-0.0628019323671497,
|
||||
0.03155339805825241
|
||||
],
|
||||
"diff": -0.01538461538461533,
|
||||
"groups": 220,
|
||||
"metric": "within_pair_auc",
|
||||
"resamples": 10000,
|
||||
"se": 0.024030780235707972,
|
||||
"units": 501
|
||||
},
|
||||
"fraction": 1.0,
|
||||
"groups": 220,
|
||||
"z": -0.6402045723740141
|
||||
}
|
||||
],
|
||||
"manifest_digest": "1b1ec814cde12801491376d2086c8c2d6db70f4a86fc308c0adc5a530398e554",
|
||||
"mde": 0.0,
|
||||
"primary": [
|
||||
{
|
||||
"name": "within_pair_auc"
|
||||
}
|
||||
],
|
||||
"reason": "the CI of within_pair_auc spans 0; the incumbent stays",
|
||||
"resamples": 10000,
|
||||
"secondary": [
|
||||
{
|
||||
"name": "top_30_precision"
|
||||
},
|
||||
{
|
||||
"name": "unsure_rate"
|
||||
},
|
||||
{
|
||||
"name": "usd_per_candidate"
|
||||
}
|
||||
],
|
||||
"spend_usd": {
|
||||
"A": 1.444503,
|
||||
"B": 1.589038
|
||||
},
|
||||
"tests": {
|
||||
"top_30_precision": {
|
||||
"ci_vs_zero": "spans",
|
||||
"direction": 1,
|
||||
"estimate": {
|
||||
"a": 1.0,
|
||||
"b": 1.0,
|
||||
"ci": [
|
||||
0.0,
|
||||
0.0
|
||||
],
|
||||
"diff": 0.0,
|
||||
"groups": 220,
|
||||
"metric": "top_30_precision",
|
||||
"resamples": 10000,
|
||||
"se": 0.0006666666666666664,
|
||||
"units": 501
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
},
|
||||
"unsure_rate": {
|
||||
"ci_vs_zero": "favour",
|
||||
"direction": -1,
|
||||
"estimate": {
|
||||
"a": 0.29141716566866266,
|
||||
"b": 0.23952095808383234,
|
||||
"ci": [
|
||||
-0.08583690987124465,
|
||||
-0.01996007984031939
|
||||
],
|
||||
"diff": -0.05189620758483032,
|
||||
"groups": 220,
|
||||
"metric": "unsure_rate",
|
||||
"resamples": 10000,
|
||||
"se": 0.01685991406098136,
|
||||
"units": 501
|
||||
},
|
||||
"mcnemar": {
|
||||
"b": 47,
|
||||
"c": 21,
|
||||
"concordant": 433,
|
||||
"p_value": 0.0021862431776861705
|
||||
},
|
||||
"target": null
|
||||
},
|
||||
"usd_per_candidate": {
|
||||
"ci_vs_zero": "against",
|
||||
"direction": -1,
|
||||
"estimate": {
|
||||
"a": 0.002883239520958083,
|
||||
"b": 0.0031717325349301397,
|
||||
"ci": [
|
||||
0.0002458205128205199,
|
||||
0.0003319800399201574
|
||||
],
|
||||
"diff": 0.0002884930139720565,
|
||||
"groups": 220,
|
||||
"metric": "usd_per_candidate",
|
||||
"resamples": 10000,
|
||||
"se": 2.1746161270931252e-05,
|
||||
"units": 501
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
},
|
||||
"within_pair_auc": {
|
||||
"ci_vs_zero": "spans",
|
||||
"direction": 1,
|
||||
"estimate": {
|
||||
"a": 0.7615384615384615,
|
||||
"b": 0.7461538461538462,
|
||||
"ci": [
|
||||
-0.0628019323671497,
|
||||
0.03155339805825241
|
||||
],
|
||||
"diff": -0.01538461538461533,
|
||||
"groups": 220,
|
||||
"metric": "within_pair_auc",
|
||||
"resamples": 10000,
|
||||
"se": 0.024030780235707972,
|
||||
"units": 501
|
||||
},
|
||||
"mcnemar": null,
|
||||
"target": null
|
||||
}
|
||||
},
|
||||
"tost": null,
|
||||
"units": {
|
||||
"by_label": {
|
||||
"0": 218,
|
||||
"1": 283
|
||||
},
|
||||
"groups": 220,
|
||||
"outcomes": 1002,
|
||||
"paired": 501
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user