mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(hunter): ask for the final answer in plain text; DeepSeek's JSON mode lost it
The final-answer turn requested the provider's JSON mode with the tools still
attached. DeepSeek (deepseek-flash) then answered with the format itself
(`{"type": "json_object"}`), wrapped the array (`{"type": "json_object",
"value": [...]}`), or emitted its tool-call markup as text. A hunt that had found
the vulnerability reported nothing.
Measured on the v2 sink verifier with the declared vulnerable functions: 0 of 7
confirmed in JSON mode, 5 of 7 asked plainly (6 of 7 at 8 steps). The model named
Medplum's actual redirect-URI prefix-match bug, LMDeploy's redirect bypass of
_is_safe_url, and ContextForge's unsandboxed Jinja2.
The final turn is now plain text without tools. The parser also reads a
`{"value": [...]}` wrapper. Two tests that pinned JSON mode now pin the plain
turn, and a new test reproduces the wrapped answer.
Invalidates as measurements: the hunter experiment on v2 (c4a0bd8) and the first
sink-verifier run. Both used this turn.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
267bca5232
commit
3ffb8b4c54
@@ -151,7 +151,9 @@ def test_a_budget_that_runs_out_mid_hunt_still_asks_for_the_answer(tmp_path: Pat
|
||||
findings = run_tool_hunter(root, [_hotspot()], client=client, model="test-hunter", max_steps=1)
|
||||
|
||||
assert [item.line for item in findings] == [item["line"] for item in _DUMP_FINDINGS]
|
||||
assert len(client.calls) == 2 and client.calls[-1].get("json_object") is True
|
||||
# The answer turn is asked plainly and without tools: DeepSeek's JSON mode lost the answers (see
|
||||
# test_the_final_answer_is_asked_plainly_and_a_wrapped_array_is_read).
|
||||
assert len(client.calls) == 2 and not client.calls[-1].get("json_object") and client.calls[-1].get("tools") == []
|
||||
|
||||
|
||||
_UNSAFE_SNIPPET = 'client = docker.DockerClient(base_url="unix://var/run/docker.sock")\n'
|
||||
@@ -331,3 +333,35 @@ def test_openrouter_hunter_client_maps_tool_calls_without_network() -> None:
|
||||
assert len(response.tool_calls) == 1
|
||||
assert response.tool_calls[0].name == "grep_repo"
|
||||
assert response.tool_calls[0].arguments == {"pattern": "eval", "max_matches": 4}
|
||||
|
||||
|
||||
def test_the_final_answer_is_asked_plainly_and_a_wrapped_array_is_read(tmp_path) -> None:
|
||||
"""In JSON mode with tools attached, DeepSeek answered `{"type": "json_object"}` or wrapped the array as
|
||||
`{"type": "json_object", "value": [...]}`; a hunt that had found the bug reported nothing."""
|
||||
import json
|
||||
|
||||
from openultrasast import tool_hunter
|
||||
from openultrasast.complexity.map import Hotspot
|
||||
|
||||
(tmp_path / "app.py").write_text("def run(q):\n return eval(q)\n")
|
||||
finding = {"path": "app.py", "line": 2, "title": "eval of request data", "rationale": "q reaches eval"}
|
||||
seen: list[dict] = []
|
||||
|
||||
class Client:
|
||||
def __init__(self) -> None:
|
||||
self.turns = [
|
||||
tool_hunter.ChatResponse(tool_calls=(tool_hunter.ToolCall(id="c1", name="read_file", arguments={"path": "app.py"}),)),
|
||||
tool_hunter.ChatResponse(content="The input reaches eval on line 2."), # prose: triggers the final turn
|
||||
tool_hunter.ChatResponse(content=json.dumps({"type": "json_object", "value": [finding]})),
|
||||
]
|
||||
|
||||
def complete(self, **kw):
|
||||
seen.append({"tools": kw.get("tools"), "json_object": kw.get("json_object", False)})
|
||||
return self.turns.pop(0)
|
||||
|
||||
spot = Hotspot(
|
||||
path="app.py", function_name="run", score=1.0, band="x", signals={}, rationale="x", test_hint=None, inventory_finding_ids=()
|
||||
)
|
||||
found = tool_hunter.run_tool_hunter(tmp_path, [spot], client=Client(), model="m", max_steps=3)
|
||||
assert [(f.path, f.line) for f in found] == [("app.py", 2)]
|
||||
assert seen[-1] == {"tools": [], "json_object": False}, seen[-1]
|
||||
|
||||
Reference in New Issue
Block a user