fix(hunt): lines read through a tool count as shown; an unparseable answer is an empty answer

The answer check counted only the rendered function text as shown, so 7 of
8 baseline suspicions citing lines the model had read with read_file were
marked unknown (baseline 1/12 against the pilot hunter's 6/12). Tools now
record the lines they return. A DSPy parse failure scores as zero findings
with a flag instead of a broken run. The per-run response cache now takes
effect through dspy.configure_cache.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-13 13:10:40 +02:00
co-authored by Claude Opus 5.5
parent c5bf52f47d
commit 602ec23e03
8 changed files with 174 additions and 62 deletions
+20
View File
@@ -95,3 +95,23 @@ def test_build_lm_needs_a_key_from_the_environment(monkeypatch):
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
with pytest.raises(RuntimeError, match="DEEPSEEK_API_KEY"):
hp.build_lm()
def test_lines_read_through_a_tool_count_as_shown(tmp_path):
(tmp_path / "app.py").write_text("a = 1\nb = 2\nc = eval(input())\n")
seen: dict[str, set[int]] = {}
tools = {tool.name: tool for tool in hp.repository_tools(tmp_path, seen=seen)}
tools["read_file"](path="app.py")
assert seen["app.py"] >= {1, 2, 3}
inputs = hp.render_rows(CHANGE, ROWS, TEXT)
finding = {"path": "app.py", "line": 3, "family": "injection", "confidence": 0.9, "rationale": "eval", "cited_lines": [3]}
assert hp.check([finding], inputs)[0]["family"] == "unknown" # line 3 is outside the rendered span 5-7
assert hp.check([finding], inputs, seen)[0]["family"] == "injection" # but the model read it through a tool
def test_an_unparseable_answer_is_an_empty_flagged_answer():
lm = DummyLM([{"findings": "not json at all"}] * 4)
dspy.configure(lm=lm, adapter=dspy.ChatAdapter())
program = hp.HuntProgram()
prediction = program(**hp.render_rows(CHANGE, ROWS, TEXT))
assert prediction.findings == [] and prediction.parse_failed is True