marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

measure: both arms over the access-control training pool (2 of 5 pushes, 4 of 48 locations each); runner books real cost

Browse Source 
DSPy baseline: 2/5 pushes, 4/48 locations, 14 of 90 ordinary pushes flagged, 25 suspicions (3 true), 11 invalid-request
failures in shopper and trigger.dev, $6.13. Pilot hunter: 2/5 pushes, 4/48 locations, 99 of 100 ordinary pushes
flagged, 594 findings (3 true). Different pushes caught; nothing in trigger.dev.

The runner booked usd 0.0 per change so --budget-usd never stopped a run; it now prices every LM call of the change
at the registered prices and writes usd into the record.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-14 01:14:46 +02:00
co-authored by Claude Fable 5.1
parent fcb43fdd47
commit dee23c23f5
7 changed files with 1090 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_hunt_runner.py
+14
View File
@@ -127,3 +127,17 @@ def test_full_checkout_restores_a_truncated_file_and_refuses_an_unread_tree(repo
monkeypatch.setattr(runner, "_git", still_dirty)
with pytest.raises(runner.InstrumentFailure, match="checkout_incomplete"):
runner._full_checkout(worktree)
def test_spent_since_prices_only_the_new_calls_of_a_registered_model():
history = [
{"model": "openai/deepseek-flash", "usage": {"prompt_tokens": 10, "completion_tokens": 10}},
{
"model": "openai/deepseek-flash",
"usage": {"prompt_tokens": 1_000_000, "completion_tokens": 100_000, "prompt_tokens_details": {"cached_tokens": 500_000}},
},
{"model": "other/unregistered", "usage": {"prompt_tokens": 1_000_000, "completion_tokens": 1_000_000}},
{"model": "openai/deepseek-flash", "usage": None},
]
assert runner.spent_since(history, 1) == pytest.approx(0.5 * 0.44 + 0.5 * 0.014 + 0.1 * 1.32)
assert runner.spent_since(None, 0) == 0.0