From abd82c6e33f9b49ea7f2c3443a7c53abed4b522d Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:47:12 +0200 Subject: [PATCH] fix(benchmark): meter the provider client before building the classifier Caller Live run: every classification failed with ValueError because Caller.ask reads client.usage as a dict (the learn CLI always hands it a MeteredClient) while a bare DeepSeekChatClient keeps usage as a list of rows. build_caller now wraps a client that carries a usage list in MeteredClient; scripted and already-metered clients pass through unchanged. Regression test runs classify_family through the real DeepSeekChatClient wrapper with a stub transport (the module fixture that stubs classify is restored for it). The error field stays the exception type only, since a message can carry a model key. Co-Authored-By: Claude Fable 5.1 --- benchmarks/valid/hunter_arm.py | 18 ++++++++++++++++- tests/test_hunter_arm.py | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/benchmarks/valid/hunter_arm.py b/benchmarks/valid/hunter_arm.py index 38eef73..8378ebe 100644 --- a/benchmarks/valid/hunter_arm.py +++ b/benchmarks/valid/hunter_arm.py @@ -18,6 +18,7 @@ from openultrasast.hunter_tools import clamp_repo_path from openultrasast.learn.program import ANSWER_FORMAT, Caller, Prompt, classify from openultrasast.model.endpoint import resolve_models from openultrasast.model.taxonomy import FAMILY_IDS, load_families +from openultrasast.plane.budget import MeteredClient if __package__ in {None, ""}: sys.path.insert(0, str(Path(__file__).resolve().parents[2])) @@ -51,6 +52,20 @@ def _hunter_family(finding): return next((tag.split(":", 1)[1].strip() for tag in finding.tags if tag.startswith("family:")), "") +def build_caller(client, model): + """The classifier's Caller over a metered client. + + Caller.ask reads `client.usage` as a dict (the learn CLI always hands it a + MeteredClient). A bare DeepSeekChatClient keeps `usage` as a list of rows, which + made every live classification fail with ValueError while stubbed tests passed. + """ + # Only the provider client keeps `usage` as a list of rows; scripted and metered clients + # already have the shape Caller.ask reads, so they pass through unchanged. + if isinstance(getattr(client, "usage", None), list): + return Caller(client=MeteredClient(client), model=model) + return Caller(client=client, model=model) + + def classify_family(finding, root, *, caller, families, context_lines=20): """Classify one suspicion from its code, without access to benchmark labels.""" try: @@ -99,6 +114,7 @@ def classify_family(finding, root, *, caller, families, context_lines=20): } except Exception as error: # noqa: BLE001 # Provider messages can contain credentials. Keep a short error type. + # Type only: an exception message can carry a model key or a URL, so it is never recorded. return {"family": "unknown", "error": type(error).__name__} @@ -227,7 +243,7 @@ def _hunt_change(change, repos, *, client, model, max_steps, no_classify=False): if not hotspots: raise InstrumentFailure("no_readable_hotspots") found = tool_hunter.run_tool_hunter(root, hotspots, client=client, model=model, max_steps=max_steps) - caller = Caller(client=client, model=model) + caller = build_caller(client, model) families = tuple(family for family in FAMILY_IDS if family != "unknown") stats = record["classifier"] for finding in found: diff --git a/tests/test_hunter_arm.py b/tests/test_hunter_arm.py index 26c5114..2e0d40a 100644 --- a/tests/test_hunter_arm.py +++ b/tests/test_hunter_arm.py @@ -497,3 +497,39 @@ def test_combine_without_hunter_index_keeps_replay(fixture, tmp_path, directory_ combined = tmp_path / "combined.json" hunter_arm.combine(replay_out, hunter_out, combined) assert score.load_records(combined) == [{**change, **a, "arms": {"a": "quick_only", "b": "missing"}}] + + +def test_build_caller_classifies_through_the_real_deepseek_wrapper(tmp_path, monkeypatch): + """Regression: a bare DeepSeekChatClient keeps `usage` as a list, which made Caller.ask raise + ValueError on every live classification. build_caller must meter the client first.""" + from types import SimpleNamespace + + from openultrasast.model.endpoint import DeepSeekChatClient + + answer = { + "verdict": "vulnerable", + "family": "access_control", + "confidence": 0.9, + "rationale": "The guard is skipped on line 2.", + "cited_lines": [2], + } + + class Raw: + def complete_chat_raw(self, **kw): + return {"choices": [{"message": {"content": json.dumps(answer)}}], "usage": {"prompt_tokens": 3}} + + (tmp_path / "app.py").write_text("def view(user):\n return secret()\n") + finding = SimpleNamespace( + path="app.py", line=2, title="guard skipped", rationale="no check", tags=["tool-hunter", "family:views"], function_name="view" + ) + # The module-wide fixture stubs classify; this test must run the real parser and Caller. + monkeypatch.setattr(hunter_arm, "classify", program.classify) + caller = hunter_arm.build_caller(DeepSeekChatClient(Raw()), "test-model") + assert isinstance(caller.client, hunter_arm.MeteredClient) + # A scripted client has no usage list and must pass through unwrapped. + assert hunter_arm.build_caller(tool_hunter.scripted_hunter_client(), "m").client is not None + assert not isinstance(hunter_arm.build_caller(tool_hunter.scripted_hunter_client(), "m").client, hunter_arm.MeteredClient) + families = tuple(family for family in hunter_arm.FAMILY_IDS if family != "unknown") + result = hunter_arm.classify_family(finding, tmp_path, caller=caller, families=families) + assert result["family"] == "access_control" and "error" not in result + assert result["verdict"] == "vulnerable" and result["cited_lines"] == [2]