feat(benchmark): combine --keep-verdicts drops hunter suspicions the classifier rejected

Opt-in filter on the classifier verdict for hunter-sourced defects; the default
keeps everything. Each combined record notes the kept verdicts and the counts.
Live check on the pilot: (a)+(b) filtered to 'vulnerable' gives recall 5/12 and
54.4 false alarms per 100 ordinary pushes, matching the scratch numbers.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/valid/hunter_arm.py

@@ -351,7 +351,7 @@ def _extend_defects(existing, extra):
     return merged
 
 
-def combine(replay_out, hunter_out, out):
+def combine(replay_out, hunter_out, out, *, keep_verdicts=None):
     """Preserve replay coverage while adding independently detected hunter defects."""
     a = _joined(replay_out)
     b = _joined(hunter_out) if (hunter_out / "inputs.json").exists() else {}
@@ -359,13 +359,30 @@ def combine(replay_out, hunter_out, out):
     for identity, left in a.items():
         right = b.get(identity, {})
         has_a, has_b = "push" in left, "push" in right
+        hunter_defects = right.get("push", {}).get("admission", {}).get("defects", [])
+        hunter_defects_total = sum(defect.get("source") == "hunter" for defect in hunter_defects)
+        kept_defects = [
+            defect
+            for defect in hunter_defects
+            if keep_verdicts is None
+            or defect.get("source") != "hunter"
+            or (defect.get("classifier") or {}).get("verdict", "unsure") in keep_verdicts
+        ]
+        hunter_defects_kept = sum(defect.get("source") == "hunter" for defect in kept_defects)
         merged = deepcopy(left if has_a or not has_b else right)
         if has_a and has_b:
             admission = merged["push"].setdefault("admission", {})
             admission["defects"] = _extend_defects(
                 admission.get("defects", []),
-                right["push"].get("admission", {}).get("defects", []),
+                kept_defects,
             )
+        elif has_b and hunter_defects_kept != hunter_defects_total:
+            merged["push"]["admission"]["defects"] = deepcopy(kept_defects)
+        merged.setdefault("combine", {}).update(
+            keep_verdicts=list(keep_verdicts) if keep_verdicts is not None else None,
+            hunter_defects_total=hunter_defects_total,
+            hunter_defects_kept=hunter_defects_kept,
+        )
         merged["arms"] = {
             "a": left.get("status", "missing") if has_a else "missing",
             "b": right.get("status", "missing") if has_b else "missing",
@@ -375,6 +392,13 @@ def combine(replay_out, hunter_out, out):
     return combined
 
 
+def _keep_verdicts(value):
+    verdicts = list(dict.fromkeys(verdict.strip() for verdict in value.split(",")))
+    if any(verdict not in ("vulnerable", "not_vulnerable", "unsure") for verdict in verdicts):
+        raise argparse.ArgumentTypeError("Expected a comma-separated list of vulnerable, not_vulnerable, unsure")
+    return verdicts
+
+
 def main(argv=None):
     parser = argparse.ArgumentParser(description=__doc__)
     commands = parser.add_subparsers(dest="command", required=True)
@@ -391,9 +415,12 @@ def main(argv=None):
     join.add_argument("replay_out", type=Path)
     join.add_argument("hunter_out", type=Path)
     join.add_argument("--out", type=Path, required=True)
+    join.add_argument(
+        "--keep-verdicts", type=_keep_verdicts, help="Keep hunter defects with these comma-separated classifier verdicts (default: all)"
+    )
     args = parser.parse_args(argv)
     if args.command == "combine":
-        combine(args.replay_out, args.hunter_out, args.out)
+        combine(args.replay_out, args.hunter_out, args.out, keep_verdicts=args.keep_verdicts)
         return 0
     return run(
         args.manifest,

tests/test_hunter_arm.py

@@ -423,6 +423,85 @@ def write_arm(out, entries, records):
         record_path(out, item_id).write_text(json.dumps(record))
 
 
+@pytest.fixture
+def combine_arms(tmp_path):
+    change = {"id": "combine-fixture", "kind": "ordinary"}
+    entry = {"item_id": change["id"], "change": change, "input_digest": "fixture-digest"}
+    replay_defect = {"defect_id": "replay", "source": "engine", "family": "injection", "locations": [["app.py", 1]]}
+    hunter_defects = [
+        {
+            "defect_id": verdict,
+            "source": "hunter",
+            "family": "injection",
+            "locations": [["app.py", line]],
+            "classifier": {"verdict": verdict},
+        }
+        for line, verdict in enumerate(("vulnerable", "not_vulnerable", "unsure"), start=2)
+    ]
+    other_defect = {"defect_id": "other", "source": "engine", "family": "path", "locations": [["app.py", 5]]}
+    a = {
+        "status": "quick_only",
+        "input_digest": entry["input_digest"],
+        "push": {"admission": {"defects": [replay_defect], "dispositions": [{"sentinel": "keep"}]}},
+        "combine": {"sentinel": "keep"},
+    }
+    b = {
+        "status": "hunter_covered",
+        "input_digest": entry["input_digest"],
+        "push": {"admission": {"defects": [*hunter_defects, other_defect]}},
+        "combine": {"sentinel": "keep"},
+    }
+    replay_out, hunter_out, combined = tmp_path / "a", tmp_path / "b", tmp_path / "combined.json"
+    write_arm(replay_out, [entry], {change["id"]: a})
+    write_arm(hunter_out, [entry], {change["id"]: b})
+    return replay_out, hunter_out, combined, entry, a, b
+
+
+def test_combine_default_keeps_all_verdicts(combine_arms):
+    replay_out, hunter_out, combined, _, a, b = combine_arms
+    assert hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined)]) == 0
+    (row,) = score.load_records(combined)
+    assert row["push"]["admission"]["defects"] == a["push"]["admission"]["defects"] + b["push"]["admission"]["defects"]
+    assert row["combine"] == {"sentinel": "keep", "keep_verdicts": None, "hunter_defects_total": 3, "hunter_defects_kept": 3}
+
+
+@pytest.mark.parametrize("with_replay", [True, False])
+def test_combine_keeps_only_vulnerable_hunter_defects(combine_arms, with_replay):
+    replay_out, hunter_out, combined, entry, a, b = combine_arms
+    if not with_replay:
+        record_path(replay_out, entry["item_id"]).unlink()
+    assert hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined), "--keep-verdicts", "vulnerable"]) == 0
+    (row,) = score.load_records(combined)
+    expected = a["push"]["admission"]["defects"] if with_replay else []
+    assert row["push"]["admission"]["defects"] == [
+        *expected,
+        b["push"]["admission"]["defects"][0],
+        b["push"]["admission"]["defects"][-1],
+    ]
+    if with_replay:
+        assert row["push"]["admission"]["dispositions"] == a["push"]["admission"]["dispositions"]
+    assert row["combine"] == {"sentinel": "keep", "keep_verdicts": ["vulnerable"], "hunter_defects_total": 3, "hunter_defects_kept": 1}
+
+
+@pytest.mark.parametrize("classification", [{}, {"classifier": None}, {"classifier": {"error": "empty_excerpt"}}])
+@pytest.mark.parametrize("keep_verdicts", ["vulnerable", "unsure,not_vulnerable"])
+def test_combine_missing_classifier_counts_as_unsure(combine_arms, classification, keep_verdicts):
+    replay_out, hunter_out, combined, entry, a, b = combine_arms
+    defect = {"defect_id": "unclassified", "source": "hunter", "family": "path", "locations": [["app.py", 6]], **classification}
+    b["push"]["admission"]["defects"] = [defect]
+    record_path(hunter_out, entry["item_id"]).write_text(json.dumps(b))
+    assert hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined), "--keep-verdicts", keep_verdicts]) == 0
+    (row,) = score.load_records(combined)
+    kept = int("unsure" in keep_verdicts.split(","))
+    assert row["push"]["admission"]["defects"] == [*a["push"]["admission"]["defects"], *([defect] if kept else [])]
+    assert row["combine"] == {
+        "sentinel": "keep",
+        "keep_verdicts": keep_verdicts.split(","),
+        "hunter_defects_total": 1,
+        "hunter_defects_kept": kept,
+    }
+
+
 def test_combine_preserves_a_and_scores_b(fixture, tmp_path):
     _, change, _, _, _ = fixture
     changes = [{**change, "id": name} for name in ("both", "a-only", "b-only")]
@@ -459,7 +538,12 @@ def test_combine_preserves_a_and_scores_b(fixture, tmp_path):
     assert both["arms"] == {"a": "quick_only", "b": "hunter_covered"}
     assert both["push"]["quick_tier"] == a["push"]["quick_tier"]
     assert both["push"]["admission"]["dispositions"] == [{"sentinel": "keep"}]
-    assert a_only == {**changes[1], **a, "arms": {"a": "quick_only", "b": "missing"}}
+    assert a_only == {
+        **changes[1],
+        **a,
+        "arms": {"a": "quick_only", "b": "missing"},
+        "combine": {"keep_verdicts": None, "hunter_defects_total": 0, "hunter_defects_kept": 0},
+    }
     assert b_only["arms"] == {"a": "missing", "b": "hunter_covered"}
     a["push"]["admission"]["defects"] = [{**defect, "defect_id": "a-defect", "source": "engine"}]
     record_path(replay_out, "replay-both").write_text(json.dumps(a))
@@ -496,7 +580,14 @@ def test_combine_without_hunter_index_keeps_replay(fixture, tmp_path, directory_
         hunter_out.mkdir()
     combined = tmp_path / "combined.json"
     hunter_arm.combine(replay_out, hunter_out, combined)
-    assert score.load_records(combined) == [{**change, **a, "arms": {"a": "quick_only", "b": "missing"}}]
+    assert score.load_records(combined) == [
+        {
+            **change,
+            **a,
+            "arms": {"a": "quick_only", "b": "missing"},
+            "combine": {"keep_verdicts": None, "hunter_defects_total": 0, "hunter_defects_kept": 0},
+        }
+    ]
 
 
 def test_build_caller_classifies_through_the_real_deepseek_wrapper(tmp_path, monkeypatch):