mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Fable 5.1
072a102b20
verify_batched.py replaces the per-candidate verifier's design: - triage per file (no tools): where does the operation's data originate; internal/constant candidates are not verified; - one 6-step hunt per file for up to 6 candidates, a verdict per operation; - two independent passes; only agreement is reported, the rest is "disputed". On the 46-candidate validation set ($1.01): - declared sites: 16 of 20 agreed, 12 of 15 cases (the per-candidate design confirmed 11). Budibase and Froxlor are new; Unleash dropped to one pass of two. winml is triaged out as "constant": a CORS literal, which the reachability question does not fit. - the 26 pgAdmin candidates that flickered between single runs: 2 triaged out, 9 agreed in both passes, 4 disputed, 13 quiet. - $0.022 per candidate for two passes vs $0.033 under the old design; on a file with several candidates, 2.3x cheaper per pass. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
37 lines
1.2 KiB
JSON
37 lines
1.2 KiB
JSON
{
|
|
"measurement": "verify_batched.py on the 46-candidate validation set (15 declared-site cases + 26 flickering pgAdmin candidates), triage on, 2 passes, 6 steps, $1.01",
|
|
"declared_sites": {
|
|
"agreed": 16,
|
|
"of": 20,
|
|
"cases_detected": 12,
|
|
"of_cases_with_a_candidate_site": 15,
|
|
"newly_found_vs_per_candidate_design": [
|
|
"budibase (both sites)",
|
|
"froxlor listing/listingCount"
|
|
],
|
|
"lost": [
|
|
"unleash fetchRetry (found in 1 of 2 passes)"
|
|
],
|
|
"missed": [
|
|
"winml (triage: constant -- a CORS literal; the reachability question does not fit config_secrets)",
|
|
"fuxa getDaqValue (1 of 2 passes)",
|
|
"froxlor validateIpAddresses (1 of 2)"
|
|
]
|
|
},
|
|
"flickering_pgadmin_26": {
|
|
"triaged_out": 2,
|
|
"agreed_in_both_passes": 9,
|
|
"disputed": 4,
|
|
"quiet": 13,
|
|
"note": "the 9 need adjudication; agreement turns run-to-run noise into a stable set"
|
|
},
|
|
"cost": {
|
|
"usd_per_candidate_two_passes": 0.022,
|
|
"old_design_two_passes": 0.0334,
|
|
"pgadmin_subset_per_candidate_two_passes": 0.0146,
|
|
"old_single_pass": 0.0167,
|
|
"note": "per-file batching: ~2.3x cheaper per pass on a file with several candidates"
|
|
},
|
|
"decision_pending": "full run of all 17 cases with this design"
|
|
}
|