mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-20 21:49:28 +00:00
measure(independent): triage + per-file batching + two-pass agreement -- 16/20 sites, flicker cut 26 -> 9, 34% cheaper
verify_batched.py replaces the per-candidate verifier's design: - triage per file (no tools): where does the operation's data originate; internal/constant candidates are not verified; - one 6-step hunt per file for up to 6 candidates, a verdict per operation; - two independent passes; only agreement is reported, the rest is "disputed". On the 46-candidate validation set ($1.01): - declared sites: 16 of 20 agreed, 12 of 15 cases (the per-candidate design confirmed 11). Budibase and Froxlor are new; Unleash dropped to one pass of two. winml is triaged out as "constant": a CORS literal, which the reachability question does not fit. - the 26 pgAdmin candidates that flickered between single runs: 2 triaged out, 9 agreed in both passes, 4 disputed, 13 quiet. - $0.022 per candidate for two passes vs $0.033 under the old design; on a file with several candidates, 2.3x cheaper per pass. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
bad8c7ada4
commit
072a102b20
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"measurement": "verify_batched.py on the 46-candidate validation set (15 declared-site cases + 26 flickering pgAdmin candidates), triage on, 2 passes, 6 steps, $1.01",
|
||||
"declared_sites": {
|
||||
"agreed": 16,
|
||||
"of": 20,
|
||||
"cases_detected": 12,
|
||||
"of_cases_with_a_candidate_site": 15,
|
||||
"newly_found_vs_per_candidate_design": [
|
||||
"budibase (both sites)",
|
||||
"froxlor listing/listingCount"
|
||||
],
|
||||
"lost": [
|
||||
"unleash fetchRetry (found in 1 of 2 passes)"
|
||||
],
|
||||
"missed": [
|
||||
"winml (triage: constant -- a CORS literal; the reachability question does not fit config_secrets)",
|
||||
"fuxa getDaqValue (1 of 2 passes)",
|
||||
"froxlor validateIpAddresses (1 of 2)"
|
||||
]
|
||||
},
|
||||
"flickering_pgadmin_26": {
|
||||
"triaged_out": 2,
|
||||
"agreed_in_both_passes": 9,
|
||||
"disputed": 4,
|
||||
"quiet": 13,
|
||||
"note": "the 9 need adjudication; agreement turns run-to-run noise into a stable set"
|
||||
},
|
||||
"cost": {
|
||||
"usd_per_candidate_two_passes": 0.022,
|
||||
"old_design_two_passes": 0.0334,
|
||||
"pgadmin_subset_per_candidate_two_passes": 0.0146,
|
||||
"old_single_pass": 0.0167,
|
||||
"note": "per-file batching: ~2.3x cheaper per pass on a file with several candidates"
|
||||
},
|
||||
"decision_pending": "full run of all 17 cases with this design"
|
||||
}
|
||||
Reference in New Issue
Block a user