fix(benchmark): meter the provider client before building the classifier Caller

Live run: every classification failed with ValueError because Caller.ask
reads client.usage as a dict (the learn CLI always hands it a MeteredClient)
while a bare DeepSeekChatClient keeps usage as a list of rows. build_caller
now wraps a client that carries a usage list in MeteredClient; scripted and
already-metered clients pass through unchanged.

Regression test runs classify_family through the real DeepSeekChatClient
wrapper with a stub transport (the module fixture that stubs classify is
restored for it). The error field stays the exception type only, since a
message can carry a model key.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/valid/hunter_arm.py

@@ -18,6 +18,7 @@ from openultrasast.hunter_tools import clamp_repo_path
 from openultrasast.learn.program import ANSWER_FORMAT, Caller, Prompt, classify
 from openultrasast.model.endpoint import resolve_models
 from openultrasast.model.taxonomy import FAMILY_IDS, load_families
+from openultrasast.plane.budget import MeteredClient
 
 if __package__ in {None, ""}:
     sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
@@ -51,6 +52,20 @@ def _hunter_family(finding):
     return next((tag.split(":", 1)[1].strip() for tag in finding.tags if tag.startswith("family:")), "")
 
 
+def build_caller(client, model):
+    """The classifier's Caller over a metered client.
+
+    Caller.ask reads `client.usage` as a dict (the learn CLI always hands it a
+    MeteredClient). A bare DeepSeekChatClient keeps `usage` as a list of rows, which
+    made every live classification fail with ValueError while stubbed tests passed.
+    """
+    # Only the provider client keeps `usage` as a list of rows; scripted and metered clients
+    # already have the shape Caller.ask reads, so they pass through unchanged.
+    if isinstance(getattr(client, "usage", None), list):
+        return Caller(client=MeteredClient(client), model=model)
+    return Caller(client=client, model=model)
+
+
 def classify_family(finding, root, *, caller, families, context_lines=20):
     """Classify one suspicion from its code, without access to benchmark labels."""
     try:
@@ -99,6 +114,7 @@ def classify_family(finding, root, *, caller, families, context_lines=20):
         }
     except Exception as error:  # noqa: BLE001
         # Provider messages can contain credentials. Keep a short error type.
+        # Type only: an exception message can carry a model key or a URL, so it is never recorded.
         return {"family": "unknown", "error": type(error).__name__}
 
 
@@ -227,7 +243,7 @@ def _hunt_change(change, repos, *, client, model, max_steps, no_classify=False):
                 if not hotspots:
                     raise InstrumentFailure("no_readable_hotspots")
                 found = tool_hunter.run_tool_hunter(root, hotspots, client=client, model=model, max_steps=max_steps)
-                caller = Caller(client=client, model=model)
+                caller = build_caller(client, model)
                 families = tuple(family for family in FAMILY_IDS if family != "unknown")
                 stats = record["classifier"]
                 for finding in found:

tests/test_hunter_arm.py

@@ -497,3 +497,39 @@ def test_combine_without_hunter_index_keeps_replay(fixture, tmp_path, directory_
     combined = tmp_path / "combined.json"
     hunter_arm.combine(replay_out, hunter_out, combined)
     assert score.load_records(combined) == [{**change, **a, "arms": {"a": "quick_only", "b": "missing"}}]
+
+
+def test_build_caller_classifies_through_the_real_deepseek_wrapper(tmp_path, monkeypatch):
+    """Regression: a bare DeepSeekChatClient keeps `usage` as a list, which made Caller.ask raise
+    ValueError on every live classification. build_caller must meter the client first."""
+    from types import SimpleNamespace
+
+    from openultrasast.model.endpoint import DeepSeekChatClient
+
+    answer = {
+        "verdict": "vulnerable",
+        "family": "access_control",
+        "confidence": 0.9,
+        "rationale": "The guard is skipped on line 2.",
+        "cited_lines": [2],
+    }
+
+    class Raw:
+        def complete_chat_raw(self, **kw):
+            return {"choices": [{"message": {"content": json.dumps(answer)}}], "usage": {"prompt_tokens": 3}}
+
+    (tmp_path / "app.py").write_text("def view(user):\n    return secret()\n")
+    finding = SimpleNamespace(
+        path="app.py", line=2, title="guard skipped", rationale="no check", tags=["tool-hunter", "family:views"], function_name="view"
+    )
+    # The module-wide fixture stubs classify; this test must run the real parser and Caller.
+    monkeypatch.setattr(hunter_arm, "classify", program.classify)
+    caller = hunter_arm.build_caller(DeepSeekChatClient(Raw()), "test-model")
+    assert isinstance(caller.client, hunter_arm.MeteredClient)
+    # A scripted client has no usage list and must pass through unwrapped.
+    assert hunter_arm.build_caller(tool_hunter.scripted_hunter_client(), "m").client is not None
+    assert not isinstance(hunter_arm.build_caller(tool_hunter.scripted_hunter_client(), "m").client, hunter_arm.MeteredClient)
+    families = tuple(family for family in hunter_arm.FAMILY_IDS if family != "unknown")
+    result = hunter_arm.classify_family(finding, tmp_path, caller=caller, families=families)
+    assert result["family"] == "access_control" and "error" not in result
+    assert result["verdict"] == "vulnerable" and result["cited_lines"] == [2]