From 3ffb8b4c5482c9280d0b02b6ecee2f7854673787 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:10:57 +0200 Subject: [PATCH] fix(hunter): ask for the final answer in plain text; DeepSeek's JSON mode lost it The final-answer turn requested the provider's JSON mode with the tools still attached. DeepSeek (deepseek-flash) then answered with the format itself (`{"type": "json_object"}`), wrapped the array (`{"type": "json_object", "value": [...]}`), or emitted its tool-call markup as text. A hunt that had found the vulnerability reported nothing. Measured on the v2 sink verifier with the declared vulnerable functions: 0 of 7 confirmed in JSON mode, 5 of 7 asked plainly (6 of 7 at 8 steps). The model named Medplum's actual redirect-URI prefix-match bug, LMDeploy's redirect bypass of _is_safe_url, and ContextForge's unsandboxed Jinja2. The final turn is now plain text without tools. The parser also reads a `{"value": [...]}` wrapper. Two tests that pinned JSON mode now pin the plain turn, and a new test reproduces the wrapped answer. Invalidates as measurements: the hunter experiment on v2 (c4a0bd8) and the first sink-verifier run. Both used this turn. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- src/openultrasast/tool_hunter.py | 11 ++++++++- tests/test_hunter_curated_tools.py | 4 +++- tests/test_tool_hunter.py | 36 +++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/openultrasast/tool_hunter.py b/src/openultrasast/tool_hunter.py index 101e9a8..c1ee2f9 100644 --- a/src/openultrasast/tool_hunter.py +++ b/src/openultrasast/tool_hunter.py @@ -396,7 +396,13 @@ def _final_answer( ), }, ] - response = client.complete(model=model, messages=messages, tools=schemas, json_object=True) + # Plain text, no tools. In JSON mode with tools attached, DeepSeek answered the final turn with the format + # itself (`{"type": "json_object"}`), wrapped the array (`{"type": "json_object", "value": [...]}`) or emitted + # its tool-call markup as text -- and a hunt that had found the bug reported nothing. Measured on the v2 sink + # verifier: 0 of 7 declared vulnerable functions confirmed in JSON mode, 5 of 7 asked plainly (6 of 7 at 8 + # steps). The parser already takes a bare or fenced array. + del schemas + response = client.complete(model=model, messages=messages, tools=[], json_object=False) return _findings_from_content(root, hotspots, response.content or "", tags) @@ -548,6 +554,9 @@ def _findings_from_content( items = payload elif isinstance(payload, dict) and isinstance(payload.get("findings"), list): items = payload["findings"] + elif isinstance(payload, dict) and isinstance(payload.get("value"), list): + # DeepSeek in JSON mode wrapped the array it was asked for as `{"type": "json_object", "value": [...]}`. + items = payload["value"] else: return [] scores = {hotspot.path: hotspot.score for hotspot in hotspots} diff --git a/tests/test_hunter_curated_tools.py b/tests/test_hunter_curated_tools.py index 838e90c..002a9a5 100644 --- a/tests/test_hunter_curated_tools.py +++ b/tests/test_hunter_curated_tools.py @@ -179,7 +179,9 @@ def test_the_loop_asks_once_more_for_the_answer_instead_of_dropping_prose() -> N findings = run_tool_hunter(root, [], client=client, model="m", max_steps=3) assert [item.line for item in findings] == [2] final = client.calls[-1] - assert final.get("json_object") is True, "the last turn asks the provider for JSON, it does not hope for it" + # Asked for JSON in the prompt, not through the provider's JSON mode: with tools attached, DeepSeek's JSON mode + # returned `{"type": "json_object"}` and lost the answers (tests/test_tool_hunter.py pins the shapes). + assert not final.get("json_object") and final.get("tools") == [], "the last turn is plain text without tools" text = "\n".join(str(message.get("content") or "") for message in final["messages"]) assert "JSON" in text and prose in text, "the prose it already wrote is what the final turn is asked to format" diff --git a/tests/test_tool_hunter.py b/tests/test_tool_hunter.py index aac1693..d524d56 100644 --- a/tests/test_tool_hunter.py +++ b/tests/test_tool_hunter.py @@ -151,7 +151,9 @@ def test_a_budget_that_runs_out_mid_hunt_still_asks_for_the_answer(tmp_path: Pat findings = run_tool_hunter(root, [_hotspot()], client=client, model="test-hunter", max_steps=1) assert [item.line for item in findings] == [item["line"] for item in _DUMP_FINDINGS] - assert len(client.calls) == 2 and client.calls[-1].get("json_object") is True + # The answer turn is asked plainly and without tools: DeepSeek's JSON mode lost the answers (see + # test_the_final_answer_is_asked_plainly_and_a_wrapped_array_is_read). + assert len(client.calls) == 2 and not client.calls[-1].get("json_object") and client.calls[-1].get("tools") == [] _UNSAFE_SNIPPET = 'client = docker.DockerClient(base_url="unix://var/run/docker.sock")\n' @@ -331,3 +333,35 @@ def test_openrouter_hunter_client_maps_tool_calls_without_network() -> None: assert len(response.tool_calls) == 1 assert response.tool_calls[0].name == "grep_repo" assert response.tool_calls[0].arguments == {"pattern": "eval", "max_matches": 4} + + +def test_the_final_answer_is_asked_plainly_and_a_wrapped_array_is_read(tmp_path) -> None: + """In JSON mode with tools attached, DeepSeek answered `{"type": "json_object"}` or wrapped the array as + `{"type": "json_object", "value": [...]}`; a hunt that had found the bug reported nothing.""" + import json + + from openultrasast import tool_hunter + from openultrasast.complexity.map import Hotspot + + (tmp_path / "app.py").write_text("def run(q):\n return eval(q)\n") + finding = {"path": "app.py", "line": 2, "title": "eval of request data", "rationale": "q reaches eval"} + seen: list[dict] = [] + + class Client: + def __init__(self) -> None: + self.turns = [ + tool_hunter.ChatResponse(tool_calls=(tool_hunter.ToolCall(id="c1", name="read_file", arguments={"path": "app.py"}),)), + tool_hunter.ChatResponse(content="The input reaches eval on line 2."), # prose: triggers the final turn + tool_hunter.ChatResponse(content=json.dumps({"type": "json_object", "value": [finding]})), + ] + + def complete(self, **kw): + seen.append({"tools": kw.get("tools"), "json_object": kw.get("json_object", False)}) + return self.turns.pop(0) + + spot = Hotspot( + path="app.py", function_name="run", score=1.0, band="x", signals={}, rationale="x", test_hint=None, inventory_finding_ids=() + ) + found = tool_hunter.run_tool_hunter(tmp_path, [spot], client=Client(), model="m", max_steps=3) + assert [(f.path, f.line) for f in found] == [("app.py", 2)] + assert seen[-1] == {"tools": [], "json_object": False}, seen[-1]