mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(learning-harness): a hunt that ends in prose is asked to format it
Measured live against DeepSeek on one pair: the model spends its step budget on tools and then answers in a paragraph that names the sink, the argument and the line -- which parses as nothing. A detector that found the bug and reported zero is indistinguishable from one that found nothing, and every family in round zero would have baselined at zero for this reason alone. The loop now takes one more turn when the hunt ends in prose or runs out of steps, asking the provider for JSON and handing back the reasoning the model already produced. It formats an answer that was reached; it never asks for a new one, and a hunt that called no tool never gets there. Same pair, same model, before and after: recall 0.00 -> 1.00, Youden 0.00 -> 1.00 over three runs per side. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
a1e7b1a20f
commit
d2ebb58007
@@ -133,7 +133,13 @@ def test_path_escape_stays_in_band_and_still_counts_as_tool_use(tmp_path: Path)
|
||||
assert "error" in json.dumps(client.calls[1]["messages"]).lower()
|
||||
|
||||
|
||||
def test_max_steps_stops_before_ungrounded_json(tmp_path: Path) -> None:
|
||||
def test_a_budget_that_runs_out_mid_hunt_still_asks_for_the_answer(tmp_path: Path) -> None:
|
||||
"""The step budget bounds the *investigation*, not the report.
|
||||
|
||||
This used to discard everything when the last step was a tool call: the loop had a tool result in hand and
|
||||
threw it away because it had no turn left to ask about it. What the budget must still refuse is a findings
|
||||
dump from a hunt that called no tool at all, which `test_hunter_dump_only_findings_are_rejected` pins.
|
||||
"""
|
||||
root = _repo_with_split_sink(tmp_path)
|
||||
client = ScriptedChatClient(
|
||||
[
|
||||
@@ -144,8 +150,8 @@ def test_max_steps_stops_before_ungrounded_json(tmp_path: Path) -> None:
|
||||
|
||||
findings = run_tool_hunter(root, [_hotspot()], client=client, model="test-hunter", max_steps=1)
|
||||
|
||||
assert findings == []
|
||||
assert len(client.calls) == 1
|
||||
assert [item.line for item in findings] == [item["line"] for item in _DUMP_FINDINGS]
|
||||
assert len(client.calls) == 2 and client.calls[-1].get("json_object") is True
|
||||
|
||||
|
||||
_UNSAFE_SNIPPET = 'client = docker.DockerClient(base_url="unix://var/run/docker.sock")\n'
|
||||
|
||||
Reference in New Issue
Block a user