From 455637bea534c787ab2bc247d408de2386f47868 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:30:13 +0200 Subject: [PATCH] fix(hunt): a malformed tool-loop step ends the loop and the answer is still extracted DSPy 3.4.1 raises AdapterParseError for an unparseable ReAct step, which is not the ValueError its loop catches, so one malformed step discarded the whole hunt (7 of 60 hunts in the repeats). The step now ends the loop like an invalid tool choice. Records carry loop_parse_failed beside parse_failed. Co-Authored-By: Claude Opus 5.5 --- src/openultrasast/hunt/program.py | 29 ++++++++++++++++++++++++++--- src/openultrasast/hunt/runner.py | 1 + tests/test_hunt_program.py | 16 ++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/openultrasast/hunt/program.py b/src/openultrasast/hunt/program.py index 2bc24e5..5cad5de 100644 --- a/src/openultrasast/hunt/program.py +++ b/src/openultrasast/hunt/program.py @@ -177,6 +177,23 @@ def repository_tools( return [dspy.Tool(read_file), dspy.Tool(grep_repo), dspy.Tool(find_refs)] +class _Loop(dspy.ReAct): + """ReAct where a step the adapter cannot parse ends the loop the way an invalid tool choice does, so the answer + is still extracted from what the loop gathered. DSPy 3.4 raises AdapterParseError, which is not the ValueError + its loop catches, so one malformed step otherwise discards the whole hunt.""" + + loop_parse_failed = False + + def _call_with_potential_trajectory_truncation(self, module, trajectory, **input_args): + try: + return super()._call_with_potential_trajectory_truncation(module, trajectory, **input_args) + except dspy.utils.exceptions.AdapterParseError as error: + if module is self.extract: + raise + self.loop_parse_failed = True + raise ValueError("unparseable loop step") from error + + class HuntProgram(dspy.Module): """One hunt over one change. With tools it is a bounded ReAct loop; without, one prediction (tests, dry runs).""" @@ -185,7 +202,7 @@ class HuntProgram(dspy.Module): self.mask = tuple(sorted(set(mask))) self.seen: dict[str, set[int]] = {} signature = signature_for(self.mask) - self.hunt = dspy.ReAct(signature, tools=list(tools), max_iters=max_iters) if tools else dspy.Predict(signature) + self.hunt = _Loop(signature, tools=list(tools), max_iters=max_iters) if tools else dspy.Predict(signature) def bind(self, root: Path) -> None: """Point the tools at one checkout and start a fresh record of what they show.""" @@ -195,13 +212,19 @@ class HuntProgram(dspy.Module): self.hunt.tools[tool.name] = tool def forward(self, **inputs: str) -> dspy.Prediction: + if isinstance(self.hunt, _Loop): + self.hunt.loop_parse_failed = False try: prediction = self.hunt(**inputs) except dspy.utils.exceptions.AdapterParseError: # An answer the program could not shape is an empty answer that scores against it, not a broken run. - return dspy.Prediction(findings=[], raw=[], parse_failed=True) + return dspy.Prediction(findings=[], raw=[], parse_failed=True, loop_parse_failed=self._loop_failed()) findings = check(getattr(prediction, "findings", []) or [], inputs, self.seen) - return dspy.Prediction(findings=findings, raw=getattr(prediction, "findings", []), parse_failed=False) + raw = getattr(prediction, "findings", []) + return dspy.Prediction(findings=findings, raw=raw, parse_failed=False, loop_parse_failed=self._loop_failed()) + + def _loop_failed(self) -> bool: + return bool(getattr(self.hunt, "loop_parse_failed", False)) def program_id(program: dspy.Module, *, mask: Iterable[str] = ()) -> str: diff --git a/src/openultrasast/hunt/runner.py b/src/openultrasast/hunt/runner.py index 034cf2e..603aaad 100644 --- a/src/openultrasast/hunt/runner.py +++ b/src/openultrasast/hunt/runner.py @@ -142,6 +142,7 @@ def hunt_change( program.bind(root) prediction = program(**inputs) record["parse_failed"] = bool(getattr(prediction, "parse_failed", False)) + record["loop_parse_failed"] = bool(getattr(prediction, "loop_parse_failed", False)) for finding in prediction.findings: if finding.get("path") in texts and not finding.get("function") and (span := _function_at(root, finding)): finding["function"] = span diff --git a/tests/test_hunt_program.py b/tests/test_hunt_program.py index 22352b1..d3b8389 100644 --- a/tests/test_hunt_program.py +++ b/tests/test_hunt_program.py @@ -115,3 +115,19 @@ def test_an_unparseable_answer_is_an_empty_flagged_answer(): program = hp.HuntProgram() prediction = program(**hp.render_rows(CHANGE, ROWS, TEXT)) assert prediction.findings == [] and prediction.parse_failed is True + + +def test_a_malformed_loop_step_ends_the_loop_and_the_answer_is_still_extracted(tmp_path): + (tmp_path / "app.py").write_text("\n".join(TEXT["app.py"]) + "\n") + answer = [{"path": "app.py", "line": 6, "family": "injection", "confidence": 0.8, "rationale": "eval", "cited_lines": [6]}] + malformed = {"next_thought": "read it", "tool_name": "read_file", "tool_args": "{}"} # history headings copied + lm = DummyLM([malformed, malformed, {"reasoning": "eval of input", "findings": json.dumps(answer)}]) + dspy.configure(lm=lm, adapter=dspy.ChatAdapter()) + program = hp.HuntProgram(tools=hp.repository_tools(tmp_path)) + program.bind(tmp_path) + prediction = program(**hp.render_rows(CHANGE, ROWS, TEXT)) + assert prediction.parse_failed is False and prediction.loop_parse_failed is True + assert [f["line"] for f in prediction.findings] == [6] + lm = DummyLM([{"next_thought": "done", "next_tool_name": "finish", "next_tool_args": "{}"}, {"reasoning": "-", "findings": "[]"}]) + dspy.configure(lm=lm, adapter=dspy.ChatAdapter()) + assert program(**hp.render_rows(CHANGE, ROWS, TEXT)).loop_parse_failed is False # reset per hunt