marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-18 12:39:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(independent): verifier flags for a known-answer check into a separate results directory

Browse Source 
--declared-sites-only, --candidates-from <jsonl> and --results-suffix, so the
efficiency change can be validated on the 180 already-verified pgAdmin candidates
and the 15 declared sites without touching the main results.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-09-29 09:52:50 +02:00
co-authored by Claude Fable 5.1
parent 041d87199a
commit 7be1f18778
1 changed files with 10 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/independent/verify_sinks.py
+10 -1
View File
@@ -136,11 +136,14 @@ def main() -> int:
parser.add_argument("--workers", type=int, default=4) parser.add_argument("--workers", type=int, default=4)
parser.add_argument("--budget-usd", type=float, default=40.0) parser.add_argument("--budget-usd", type=float, default=40.0)
parser.add_argument("--fixed", action="store_true", help="recheck detected sites at the fixed pin") parser.add_argument("--fixed", action="store_true", help="recheck detected sites at the fixed pin")
parser.add_argument("--results-suffix", default="verified", help="results go to independent-<vN>-<suffix>/")
parser.add_argument("--declared-sites-only", action="store_true", help="verify only the case's declared sites (a known-answer check)")
parser.add_argument("--candidates-from", type=Path, help="verify exactly the candidates recorded in this per-case .jsonl")
args = parser.parse_args() args = parser.parse_args()
load_config() load_config()
evaluate.use(args.population) evaluate.use(args.population)
candidates_dir = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-modelsinks") candidates_dir = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-modelsinks")
out_dir = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-verified") / "scans" out_dir = evaluate.RESULTS.with_name(evaluate.RESULTS.name + "-" + args.results_suffix) / "scans"
out_dir.mkdir(parents=True, exist_ok=True) out_dir.mkdir(parents=True, exist_ok=True)
only = {c.strip() for c in args.only.split(",") if c.strip()} only = {c.strip() for c in args.only.split(",") if c.strip()}
spent = 0.0 spent = 0.0
@@ -156,6 +159,12 @@ def main() -> int:
spent += sum(float(d.get("usd", 0.0)) for d in done) spent += sum(float(d.get("usd", 0.0)) for d in done)
record = json.loads((candidates_dir / f"{case['id']}.json").read_text()) record = json.loads((candidates_dir / f"{case['id']}.json").read_text())
todo = list({(p, fn): (p, fn, ln) for p, fn, ln in sorted(record["candidates"])}.values()) # one question per function todo = list({(p, fn): (p, fn, ln) for p, fn, ln in sorted(record["candidates"])}.values()) # one question per function
if args.declared_sites_only:
sites = {tuple(site.split("::", 1)) for site in case.get("sites", [])}
todo = [c for c in todo if (c[0], c[1]) in sites]
if args.candidates_from:
wanted = {tuple(json.loads(row)["candidate"][:2]) for row in args.candidates_from.read_text().splitlines() if row.strip()}
todo = [c for c in todo if (c[0], c[1]) in wanted]
if args.fixed: if args.fixed:
vulnerable = json.loads((out_dir / f"{case['id']}--vulnerable_a.json").read_text()) vulnerable = json.loads((out_dir / f"{case['id']}--vulnerable_a.json").read_text())
detected = {f["candidate"] for f in vulnerable["findings"]} detected = {f["candidate"] for f in vulnerable["findings"]}