From 23aeeada103af8eb6566e6f8bc6d7a973d1962ff Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:21:59 +0200 Subject: [PATCH] feat(benchmark): combine --keep-verdicts drops hunter suspicions the classifier rejected Opt-in filter on the classifier verdict for hunter-sourced defects; the default keeps everything. Each combined record notes the kept verdicts and the counts. Live check on the pilot: (a)+(b) filtered to 'vulnerable' gives recall 5/12 and 54.4 false alarms per 100 ordinary pushes, matching the scratch numbers. Co-Authored-By: Claude Fable 5.1 --- benchmarks/valid/hunter_arm.py | 33 ++++++++++-- tests/test_hunter_arm.py | 95 +++++++++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 5 deletions(-) diff --git a/benchmarks/valid/hunter_arm.py b/benchmarks/valid/hunter_arm.py index 8378ebe..beb0080 100644 --- a/benchmarks/valid/hunter_arm.py +++ b/benchmarks/valid/hunter_arm.py @@ -351,7 +351,7 @@ def _extend_defects(existing, extra): return merged -def combine(replay_out, hunter_out, out): +def combine(replay_out, hunter_out, out, *, keep_verdicts=None): """Preserve replay coverage while adding independently detected hunter defects.""" a = _joined(replay_out) b = _joined(hunter_out) if (hunter_out / "inputs.json").exists() else {} @@ -359,13 +359,30 @@ def combine(replay_out, hunter_out, out): for identity, left in a.items(): right = b.get(identity, {}) has_a, has_b = "push" in left, "push" in right + hunter_defects = right.get("push", {}).get("admission", {}).get("defects", []) + hunter_defects_total = sum(defect.get("source") == "hunter" for defect in hunter_defects) + kept_defects = [ + defect + for defect in hunter_defects + if keep_verdicts is None + or defect.get("source") != "hunter" + or (defect.get("classifier") or {}).get("verdict", "unsure") in keep_verdicts + ] + hunter_defects_kept = sum(defect.get("source") == "hunter" for defect in kept_defects) merged = deepcopy(left if has_a or not has_b else right) if has_a and has_b: admission = merged["push"].setdefault("admission", {}) admission["defects"] = _extend_defects( admission.get("defects", []), - right["push"].get("admission", {}).get("defects", []), + kept_defects, ) + elif has_b and hunter_defects_kept != hunter_defects_total: + merged["push"]["admission"]["defects"] = deepcopy(kept_defects) + merged.setdefault("combine", {}).update( + keep_verdicts=list(keep_verdicts) if keep_verdicts is not None else None, + hunter_defects_total=hunter_defects_total, + hunter_defects_kept=hunter_defects_kept, + ) merged["arms"] = { "a": left.get("status", "missing") if has_a else "missing", "b": right.get("status", "missing") if has_b else "missing", @@ -375,6 +392,13 @@ def combine(replay_out, hunter_out, out): return combined +def _keep_verdicts(value): + verdicts = list(dict.fromkeys(verdict.strip() for verdict in value.split(","))) + if any(verdict not in ("vulnerable", "not_vulnerable", "unsure") for verdict in verdicts): + raise argparse.ArgumentTypeError("Expected a comma-separated list of vulnerable, not_vulnerable, unsure") + return verdicts + + def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) @@ -391,9 +415,12 @@ def main(argv=None): join.add_argument("replay_out", type=Path) join.add_argument("hunter_out", type=Path) join.add_argument("--out", type=Path, required=True) + join.add_argument( + "--keep-verdicts", type=_keep_verdicts, help="Keep hunter defects with these comma-separated classifier verdicts (default: all)" + ) args = parser.parse_args(argv) if args.command == "combine": - combine(args.replay_out, args.hunter_out, args.out) + combine(args.replay_out, args.hunter_out, args.out, keep_verdicts=args.keep_verdicts) return 0 return run( args.manifest, diff --git a/tests/test_hunter_arm.py b/tests/test_hunter_arm.py index 2e0d40a..4275381 100644 --- a/tests/test_hunter_arm.py +++ b/tests/test_hunter_arm.py @@ -423,6 +423,85 @@ def write_arm(out, entries, records): record_path(out, item_id).write_text(json.dumps(record)) +@pytest.fixture +def combine_arms(tmp_path): + change = {"id": "combine-fixture", "kind": "ordinary"} + entry = {"item_id": change["id"], "change": change, "input_digest": "fixture-digest"} + replay_defect = {"defect_id": "replay", "source": "engine", "family": "injection", "locations": [["app.py", 1]]} + hunter_defects = [ + { + "defect_id": verdict, + "source": "hunter", + "family": "injection", + "locations": [["app.py", line]], + "classifier": {"verdict": verdict}, + } + for line, verdict in enumerate(("vulnerable", "not_vulnerable", "unsure"), start=2) + ] + other_defect = {"defect_id": "other", "source": "engine", "family": "path", "locations": [["app.py", 5]]} + a = { + "status": "quick_only", + "input_digest": entry["input_digest"], + "push": {"admission": {"defects": [replay_defect], "dispositions": [{"sentinel": "keep"}]}}, + "combine": {"sentinel": "keep"}, + } + b = { + "status": "hunter_covered", + "input_digest": entry["input_digest"], + "push": {"admission": {"defects": [*hunter_defects, other_defect]}}, + "combine": {"sentinel": "keep"}, + } + replay_out, hunter_out, combined = tmp_path / "a", tmp_path / "b", tmp_path / "combined.json" + write_arm(replay_out, [entry], {change["id"]: a}) + write_arm(hunter_out, [entry], {change["id"]: b}) + return replay_out, hunter_out, combined, entry, a, b + + +def test_combine_default_keeps_all_verdicts(combine_arms): + replay_out, hunter_out, combined, _, a, b = combine_arms + assert hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined)]) == 0 + (row,) = score.load_records(combined) + assert row["push"]["admission"]["defects"] == a["push"]["admission"]["defects"] + b["push"]["admission"]["defects"] + assert row["combine"] == {"sentinel": "keep", "keep_verdicts": None, "hunter_defects_total": 3, "hunter_defects_kept": 3} + + +@pytest.mark.parametrize("with_replay", [True, False]) +def test_combine_keeps_only_vulnerable_hunter_defects(combine_arms, with_replay): + replay_out, hunter_out, combined, entry, a, b = combine_arms + if not with_replay: + record_path(replay_out, entry["item_id"]).unlink() + assert hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined), "--keep-verdicts", "vulnerable"]) == 0 + (row,) = score.load_records(combined) + expected = a["push"]["admission"]["defects"] if with_replay else [] + assert row["push"]["admission"]["defects"] == [ + *expected, + b["push"]["admission"]["defects"][0], + b["push"]["admission"]["defects"][-1], + ] + if with_replay: + assert row["push"]["admission"]["dispositions"] == a["push"]["admission"]["dispositions"] + assert row["combine"] == {"sentinel": "keep", "keep_verdicts": ["vulnerable"], "hunter_defects_total": 3, "hunter_defects_kept": 1} + + +@pytest.mark.parametrize("classification", [{}, {"classifier": None}, {"classifier": {"error": "empty_excerpt"}}]) +@pytest.mark.parametrize("keep_verdicts", ["vulnerable", "unsure,not_vulnerable"]) +def test_combine_missing_classifier_counts_as_unsure(combine_arms, classification, keep_verdicts): + replay_out, hunter_out, combined, entry, a, b = combine_arms + defect = {"defect_id": "unclassified", "source": "hunter", "family": "path", "locations": [["app.py", 6]], **classification} + b["push"]["admission"]["defects"] = [defect] + record_path(hunter_out, entry["item_id"]).write_text(json.dumps(b)) + assert hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined), "--keep-verdicts", keep_verdicts]) == 0 + (row,) = score.load_records(combined) + kept = int("unsure" in keep_verdicts.split(",")) + assert row["push"]["admission"]["defects"] == [*a["push"]["admission"]["defects"], *([defect] if kept else [])] + assert row["combine"] == { + "sentinel": "keep", + "keep_verdicts": keep_verdicts.split(","), + "hunter_defects_total": 1, + "hunter_defects_kept": kept, + } + + def test_combine_preserves_a_and_scores_b(fixture, tmp_path): _, change, _, _, _ = fixture changes = [{**change, "id": name} for name in ("both", "a-only", "b-only")] @@ -459,7 +538,12 @@ def test_combine_preserves_a_and_scores_b(fixture, tmp_path): assert both["arms"] == {"a": "quick_only", "b": "hunter_covered"} assert both["push"]["quick_tier"] == a["push"]["quick_tier"] assert both["push"]["admission"]["dispositions"] == [{"sentinel": "keep"}] - assert a_only == {**changes[1], **a, "arms": {"a": "quick_only", "b": "missing"}} + assert a_only == { + **changes[1], + **a, + "arms": {"a": "quick_only", "b": "missing"}, + "combine": {"keep_verdicts": None, "hunter_defects_total": 0, "hunter_defects_kept": 0}, + } assert b_only["arms"] == {"a": "missing", "b": "hunter_covered"} a["push"]["admission"]["defects"] = [{**defect, "defect_id": "a-defect", "source": "engine"}] record_path(replay_out, "replay-both").write_text(json.dumps(a)) @@ -496,7 +580,14 @@ def test_combine_without_hunter_index_keeps_replay(fixture, tmp_path, directory_ hunter_out.mkdir() combined = tmp_path / "combined.json" hunter_arm.combine(replay_out, hunter_out, combined) - assert score.load_records(combined) == [{**change, **a, "arms": {"a": "quick_only", "b": "missing"}}] + assert score.load_records(combined) == [ + { + **change, + **a, + "arms": {"a": "quick_only", "b": "missing"}, + "combine": {"keep_verdicts": None, "hunter_defects_total": 0, "hunter_defects_kept": 0}, + } + ] def test_build_caller_classifies_through_the_real_deepseek_wrapper(tmp_path, monkeypatch):