marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 11:49:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

search-with-proof: per-search fair-share pilot budget (stop one search starving the rest)

Browse Source 
--ceiling-usd was one shared SpendBudget passed to every search, so an expensive
search drained the pool and starved later ones: in the last 2-pair/$2 run the first
search spent $1.60 and the second hit the ceiling at $0.34 after finishing its analysis.
The driver now allocates a dynamic fair share per search (remaining_ceiling /
searches_not_yet_started), rolls unused settled budget forward, caps each search at its
share, and keeps total settled <= ceiling. A search now gets room to author and verify
after it finds the path.

Driver-only change (benchmarks/search/pilot_run.py) plus tests. tests/test_search_pilot_run.py
39 passed; ruff clean; mypy unchanged (pre-existing benchmarks errors only).

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-07 08:35:24 +02:00
co-authored by Claude Opus 4.8
parent e8e6637833
commit 2c0c45fd1f
2 changed files with 79 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/search/pilot_run.py
+6 -3
View File
@@ -419,7 +419,7 @@ def main(argv=None):
args = parser.parse_args(argv)
if not math.isfinite(args.ceiling_usd) or args.ceiling_usd < 0:
parser.error("ceiling must be finite and nonnegative")
run_budget = SpendBudget(args.ceiling_usd)
remaining_ceiling = args.ceiling_usd
if not re.fullmatch(r"\d+(,\d+)*", args.pairs) or args.max_tokens < 1:
parser.error("pairs must be comma-separated nonnegative indices; max tokens must be positive")
indices = [int(i) for i in args.pairs.split(",")]
@@ -522,17 +522,20 @@ def main(argv=None):
)
except Exception as exc:
aborted_reason = exception_reason(exc)
for index in indices if aborted_reason is None else []:
for position, index in enumerate(indices if aborted_reason is None else []):
local = root / str(index)
local.mkdir()
if args.dry_run:
sides, demos = materialise(local / "probe", "path")
client = scripted(load_demo(demos["real"]))
pilot = Pilot(args, rows[index], index, local, store, run_budget, client, prices, lane)
searches_remaining = len(indices) - position
per_search = remaining_ceiling / searches_remaining if searches_remaining > 0 and remaining_ceiling > 0 else 0.0
pilot = Pilot(args, rows[index], index, local, store, SpendBudget(per_search), client, prices, lane)
if args.dry_run:
pilot.sides = sides
pilot.record["input_bytes"] = len((sides[0 if args.side == "vulnerable" else 1].checkout / "app.py").read_bytes())
record = pilot.run()
remaining_ceiling = max(0.0, remaining_ceiling - record["spend"]["settled"])
records.append(record)
output.write(json.dumps(record, allow_nan=False) + "\n")
output.flush()