docs(benchmark): where the hunter's false alarms come from

Sweep over the pilot's hunter records: most 'vulnerable' suspicions on ordinary
pushes sit far from the diff. Scope to the change, then confidence, then the
config_secrets family; 5/12 recall kept at 8.8 false alarms per 100 (in-sample).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/measurements/2026-10-12-valid-benchmark-pilot/NOTES.md

@@ -78,6 +78,27 @@ strict scorer; note them.
   prompt to the taxonomy (`tool_hunter.py`) so the push path emits taxonomy families natively.
 - The instrument failure: `matrix` 7018468e, no readable hotspot (`hook_bytes_read: 0`).
 
+### Where the hunter's false alarms come from (sweep over the same records, no new model calls)
+
+Of the 343 ordinary-push suspicions the classifier called `vulnerable`, only 40 sit on a line the
+push changed and only 93 within 10 lines of one. The hunter searches the whole changed file, so on
+an ordinary push it re-reports the repo's pre-existing code. Scoping to the change is the biggest
+lever, confidence the second, the `config_secrets` family the third. In-sample on 12 positives, so
+treat the table as direction, not a tuned threshold:
+
+| Filter on hunter suspicions | Recall | False alarms per 100 |
+|---|---|---|
+| raw | 6/12 | 92.5 |
+| classifier `vulnerable` | 5/12 | 54.4 |
+| `vulnerable`, confidence >= 0.8 | 5/12 | 31.8 |
+| `vulnerable`, within 10 lines of the diff | 5/12 | 22.6 |
+| `vulnerable`, confidence >= 0.8, within 10 lines | 5/12 | 11.3 |
+| `vulnerable`, confidence >= 0.8, within 10 lines, not `config_secrets` | 5/12 | 8.8 |
+
+The exact-line scope (0 lines) drops recall to 1/12 because the benchmark's "head" is the code
+before the fix, where most fixes add lines rather than change them; a developer's own push has the
+vulnerable lines in the diff by construction. Proof stays the last step, on what survives this.
+
 ### What to do with this
 
 1. The push hook needs the LLM tier (owned by `pre-push-safety-net`): the hook as shipped caught

benchmarks/measurements/2026-10-12-valid-benchmark-pilot/filter_sweep.py

@@ -0,0 +1,109 @@
+"""Sweep candidate precision filters over the pilot's hunter records (no model calls).
+
+For every hunter defect: classifier verdict, confidence, taxonomy family, and whether its
+location sits on a line the push changed (git diff base..head, head side). Then score each
+filter with the real scorer: recall over the 12 positives, false alarms per 100 ordinary pushes.
+"""
+import copy
+import json
+import re
+import subprocess
+import sys
+from collections import Counter
+from pathlib import Path
+
+sys.path.insert(0, "/home/mc/Source/OpenUltraSAST")
+from benchmarks.unseen import score  # noqa: E402
+
+P = Path("/home/mc/ousast-results/valid/pilot")
+repos = json.load(open(P / "repos_hist.json"))
+rows = score.load_records(P / "hunter")
+HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
+
+
+def changed_lines(repo, base, head):
+    out = subprocess.run(
+        ["git", "-C", repos[repo], "diff", "-U0", f"{base}..{head}", "--", "."],
+        capture_output=True, text=True, check=False,
+    ).stdout
+    lines, path = {}, None
+    for line in out.splitlines():
+        if line.startswith("+++ b/"):
+            path = line[6:]
+        elif line.startswith("@@") and path:
+            m = HUNK.match(line)
+            start, n = int(m.group(1)), int(m.group(2) or 1)
+            lines.setdefault(path, set()).update(range(start, start + max(n, 1)))
+    return lines
+
+
+def near_changed(defect, lines, slack):
+    for path, line in defect.get("locations", []):
+        hits = lines.get(path)
+        if hits and any(abs(line - h) <= slack for h in hits):
+            return True
+    return False
+
+
+def filtered(row, keep):
+    r = copy.deepcopy(row)
+    adm = r.setdefault("push", {}).setdefault("admission", {})
+    adm["defects"] = [d for d in adm.get("defects", []) if keep(d)]
+    return r
+
+
+def rate(rows_, keep):
+    pos = [r for r in rows_ if r["kind"] != "ordinary"]
+    ord_ = [r for r in rows_ if r["kind"] == "ordinary" and r.get("status") != "instrument_failure"]
+    caught = sum(bool(score.catches(filtered(r, keep))) for r in pos)
+    alarms = sum(bool(list(score.findings(filtered(r, keep)))) for r in ord_)
+    flagged = alarms + sum(bool(list(score.findings(filtered(r, keep)))) for r in pos)
+    return caught, len(pos), alarms, len(ord_), flagged
+
+
+# annotate defects with changed-line proximity
+for r in rows:
+    lines = changed_lines(r["repo"], r["base"], r["head"])
+    for d in r.get("push", {}).get("admission", {}).get("defects", []):
+        d["_changed0"] = near_changed(d, lines, 0)
+        d["_changed10"] = near_changed(d, lines, 10)
+
+C = lambda d: d.get("classifier") or {}  # noqa: E731
+V = lambda d: C(d).get("verdict")  # noqa: E731
+conf = lambda d: C(d).get("confidence") or 0  # noqa: E731
+
+filters = {
+    "raw (all suspicions)": lambda d: True,
+    "verdict=vulnerable": lambda d: V(d) == "vulnerable",
+    "vulnerable, conf>=0.7": lambda d: V(d) == "vulnerable" and conf(d) >= 0.7,
+    "vulnerable, conf>=0.8": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8,
+    "vulnerable, conf>=0.85": lambda d: V(d) == "vulnerable" and conf(d) >= 0.85,
+    "vulnerable, not config_secrets": lambda d: V(d) == "vulnerable" and d.get("family") != "config_secrets",
+    "vulnerable, conf>=0.8, not config_secrets": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8 and d.get("family") != "config_secrets",
+    "vulnerable, on a changed line": lambda d: V(d) == "vulnerable" and d["_changed0"],
+    "vulnerable, within 10 of a changed line": lambda d: V(d) == "vulnerable" and d["_changed10"],
+    "vulnerable, conf>=0.7, within 10 of changed": lambda d: V(d) == "vulnerable" and conf(d) >= 0.7 and d["_changed10"],
+    "vulnerable, conf>=0.8, within 10 of changed": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8 and d["_changed10"],
+    "vulnerable, conf>=0.8, within 10, not config_secrets": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8 and d["_changed10"] and d.get("family") != "config_secrets",
+}
+print(f"{'filter':58} recall     false alarms /100   flagged pushes")
+for name, keep in filters.items():
+    c, n, a, m, f = rate(rows, keep)
+    print(f"{name:58} {c:2}/{n:<2}      {a:3}/{m} = {100*a/m:5.1f}      {f}")
+
+# where do ordinary-push suspicions sit relative to the diff?
+tot = Counter()
+for r in rows:
+    if r["kind"] != "ordinary":
+        continue
+    for d in r.get("push", {}).get("admission", {}).get("defects", []):
+        if V(d) == "vulnerable":
+            tot["vulnerable total"] += 1
+            tot["on changed line"] += d["_changed0"]
+            tot["within 10"] += d["_changed10"]
+            tot[f"family {d.get('family')}"] += 1
+print("\nordinary-push suspicions with verdict=vulnerable:", dict(tot))
+confs = Counter(round(conf(d), 2) for r in rows if r["kind"] == "ordinary" for d in r.get("push", {}).get("admission", {}).get("defects", []) if V(d) == "vulnerable")
+print("confidence histogram (ordinary, vulnerable):", dict(sorted(confs.items())))
+confs = Counter(round(conf(d), 2) for r in rows if r["kind"] != "ordinary" for d in r.get("push", {}).get("admission", {}).get("defects", []) if V(d) == "vulnerable" and score.catches(filtered(r, lambda x, d=d: x is d)))
+print("confidence of the defects that score a catch:", dict(sorted(confs.items())))