mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(hunt): a malformed tool-loop step ends the loop and the answer is still extracted
DSPy 3.4.1 raises AdapterParseError for an unparseable ReAct step, which is not the ValueError its loop catches, so one malformed step discarded the whole hunt (7 of 60 hunts in the repeats). The step now ends the loop like an invalid tool choice. Records carry loop_parse_failed beside parse_failed. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e07ab2ac93
commit
455637bea5
@@ -115,3 +115,19 @@ def test_an_unparseable_answer_is_an_empty_flagged_answer():
|
||||
program = hp.HuntProgram()
|
||||
prediction = program(**hp.render_rows(CHANGE, ROWS, TEXT))
|
||||
assert prediction.findings == [] and prediction.parse_failed is True
|
||||
|
||||
|
||||
def test_a_malformed_loop_step_ends_the_loop_and_the_answer_is_still_extracted(tmp_path):
|
||||
(tmp_path / "app.py").write_text("\n".join(TEXT["app.py"]) + "\n")
|
||||
answer = [{"path": "app.py", "line": 6, "family": "injection", "confidence": 0.8, "rationale": "eval", "cited_lines": [6]}]
|
||||
malformed = {"next_thought": "read it", "tool_name": "read_file", "tool_args": "{}"} # history headings copied
|
||||
lm = DummyLM([malformed, malformed, {"reasoning": "eval of input", "findings": json.dumps(answer)}])
|
||||
dspy.configure(lm=lm, adapter=dspy.ChatAdapter())
|
||||
program = hp.HuntProgram(tools=hp.repository_tools(tmp_path))
|
||||
program.bind(tmp_path)
|
||||
prediction = program(**hp.render_rows(CHANGE, ROWS, TEXT))
|
||||
assert prediction.parse_failed is False and prediction.loop_parse_failed is True
|
||||
assert [f["line"] for f in prediction.findings] == [6]
|
||||
lm = DummyLM([{"next_thought": "done", "next_tool_name": "finish", "next_tool_args": "{}"}, {"reasoning": "-", "findings": "[]"}])
|
||||
dspy.configure(lm=lm, adapter=dspy.ChatAdapter())
|
||||
assert program(**hp.render_rows(CHANGE, ROWS, TEXT)).loop_parse_failed is False # reset per hunt
|
||||
|
||||
Reference in New Issue
Block a user