From 8794884a644f0a0ff01548cb5dd88aef2521afbe Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:15:14 +0200 Subject: [PATCH] fix(hunter): bound every tool result; show the provider's reason for an HTTP error FUXA failed every hunt with HTTP 400, and neither run could say why. The client discarded the response body. With the body shown, the reason was "This model's maximum context length...": FUXA ships minified bundles whose single lines run to megabytes, and grep_repo returned a matched line whole. - grep_repo and find_refs quote a matched line (240 characters), never the whole line. - Every tool result is capped at 16 x max_chars, with a truncation marker. - OpenRouterChatClient errors carry the provider's response body (first 500 characters). That would also have exposed the rejected model id in one call instead of eighteen days. After the fix, FUXA candidates verify normally at $0.004-0.013 each. A test covers the megabyte-line case. verify_sinks.py: skip futures cancelled at the spend ceiling instead of reading them (the run crashed with CancelledError). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- benchmarks/independent/verify_sinks.py | 2 ++ src/openultrasast/hunter_tools.py | 6 ++++-- src/openultrasast/provider/openrouter.py | 11 ++++++++++- src/openultrasast/tool_hunter.py | 6 +++++- tests/test_hunter_tools.py | 11 +++++++++++ 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/benchmarks/independent/verify_sinks.py b/benchmarks/independent/verify_sinks.py index 31248ee..d63b289 100644 --- a/benchmarks/independent/verify_sinks.py +++ b/benchmarks/independent/verify_sinks.py @@ -128,6 +128,8 @@ def main() -> int: for candidate in pending: futures.append(pool.submit(verify, str(checkout), list(candidate), case["family"], args.model, args.max_steps)) for future in concurrent.futures.as_completed(futures): + if future.cancelled(): # cancelled at the ceiling: never run, nothing to record + continue result = future.result() spent += float(result.get("usd", 0.0)) sink.write(json.dumps(result) + "\n") diff --git a/src/openultrasast/hunter_tools.py b/src/openultrasast/hunter_tools.py index fd5152a..ec348a6 100644 --- a/src/openultrasast/hunter_tools.py +++ b/src/openultrasast/hunter_tools.py @@ -32,6 +32,8 @@ def read_file(root: Path, path: str, *, max_chars: int) -> str: return clamped.read_text(errors="ignore")[:max_chars] +# A matched line is quoted, not returned whole: a minified bundle's single line can be megabytes. +_LINE_CHARS = 240 _Lines = tuple[tuple[str, tuple[str, ...]], ...] _SOURCE_MEMO: dict[tuple[str, int], tuple[_Lines, tuple[tuple[Path, int], ...]]] = {} @@ -72,7 +74,7 @@ def grep_repo(root: Path, pattern: str, *, max_matches: int) -> list[dict[str, o for relative, lines in _source_lines(root): for line_no, line in enumerate(lines, start=1): if compiled.search(line): - matches.append({"path": relative, "line": line_no, "text": line}) + matches.append({"path": relative, "line": line_no, "text": line[:_LINE_CHARS]}) if len(matches) >= max_matches: return matches return matches @@ -102,7 +104,7 @@ def find_refs(root: Path, symbol: str, mapping_index: object) -> list[dict[str, for relative, lines in _source_lines(resolved_root): for line_no, line in enumerate(lines, start=1): if compiled.search(line): - refs.append({"path": relative, "name": symbol, "line": line_no, "text": line, "source": "text"}) + refs.append({"path": relative, "name": symbol, "line": line_no, "text": line[:_LINE_CHARS], "source": "text"}) return refs diff --git a/src/openultrasast/provider/openrouter.py b/src/openultrasast/provider/openrouter.py index c2e6551..1c8436b 100644 --- a/src/openultrasast/provider/openrouter.py +++ b/src/openultrasast/provider/openrouter.py @@ -114,7 +114,16 @@ class OpenRouterChatClient: try: return call_with_retry(_do, attempts=self.max_attempts, base_delay=self.retry_base_delay) except (urllib.error.URLError, TimeoutError, http.client.HTTPException, json.JSONDecodeError) as exc: - raise OpenRouterError(f"OpenRouter chat request failed: {type(exc).__name__}: {exc}") from exc + # The provider's own reason is in the body. Without it a 400 said nothing: DeepSeek's "the supported + # model names are ..." cost eighteen days of silently failed calls, and a case whose every request was + # rejected could not be diagnosed at all. + detail = "" + if isinstance(exc, urllib.error.HTTPError): + try: + detail = " -- " + exc.read().decode("utf-8", errors="replace")[:500] + except Exception: # noqa: BLE001 -- a body we cannot read is still an error with a status + detail = "" + raise OpenRouterError(f"OpenRouter chat request failed: {type(exc).__name__}: {exc}{detail}") from exc def complete_json(self, *, model: str, messages: list[dict[str, str]], timeout_seconds: int = 60) -> object: message = self.complete_chat(model=model, messages=messages, timeout_seconds=timeout_seconds) diff --git a/src/openultrasast/tool_hunter.py b/src/openultrasast/tool_hunter.py index c1ee2f9..8f60384 100644 --- a/src/openultrasast/tool_hunter.py +++ b/src/openultrasast/tool_hunter.py @@ -486,7 +486,11 @@ def _run_tool(root: Path, call: ToolCall, max_chars: int = _DEFAULT_MAX_CHARS) - payload = _dispatch_tool(root, call.name, _arguments(call.arguments), max_chars) except Exception as exc: # noqa: BLE001 — tool failures stay in-band so the scan continues payload = {"error": f"{type(exc).__name__}: {exc}"} - return json.dumps(payload, sort_keys=True) + text = json.dumps(payload, sort_keys=True) + # Bounded, whatever the tool. FUXA ships minified bundles whose single lines run to megabytes; one grep match + # returned such a line whole and every later request of the hunt failed with "maximum context length". + limit = 16 * max_chars + return text if len(text) <= limit else text[:limit] + f"... [truncated {len(text) - limit} characters]" def _dispatch_tool(root: Path, name: str, arguments: Mapping[str, object], default_max_chars: int = _DEFAULT_MAX_CHARS) -> object: diff --git a/tests/test_hunter_tools.py b/tests/test_hunter_tools.py index d446ba4..0d075c9 100644 --- a/tests/test_hunter_tools.py +++ b/tests/test_hunter_tools.py @@ -143,3 +143,14 @@ def test_grep_reads_the_tree_once_and_sees_an_edit(tmp_path, monkeypatch) -> Non stamp = source.stat().st_mtime_ns + 10_000_000 os.utime(source, ns=(stamp, stamp)) assert [m["line"] for m in hunter_tools.grep_repo(tmp_path, r"query\(", max_matches=5)] == [3], "a stale read survived an edit" + + +def test_a_match_in_a_minified_line_is_quoted_not_returned_whole(tmp_path) -> None: + """FUXA's minified bundles have megabyte lines; one match overflowed the model's context for the whole hunt.""" + from openultrasast import hunter_tools, tool_hunter + + (tmp_path / "bundle.js").write_text("var a=1;" + "x" * 500_000 + ";eval(q);\n") + [match] = hunter_tools.grep_repo(tmp_path, r"var a", max_matches=5) + assert len(match["text"]) <= 240 + call = tool_hunter.ToolCall(id="c", name="grep_repo", arguments={"pattern": "x+", "max_matches": 50}) + assert len(tool_hunter._run_tool(tmp_path, call, 4000)) <= 16 * 4000 + 64