mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
docs(benchmark): where the hunter's false alarms come from
Sweep over the pilot's hunter records: most 'vulnerable' suspicions on ordinary pushes sit far from the diff. Scope to the change, then confidence, then the config_secrets family; 5/12 recall kept at 8.8 false alarms per 100 (in-sample). Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
23aeeada10
commit
ad20405fc2
@@ -78,6 +78,27 @@ strict scorer; note them.
|
||||
prompt to the taxonomy (`tool_hunter.py`) so the push path emits taxonomy families natively.
|
||||
- The instrument failure: `matrix` 7018468e, no readable hotspot (`hook_bytes_read: 0`).
|
||||
|
||||
### Where the hunter's false alarms come from (sweep over the same records, no new model calls)
|
||||
|
||||
Of the 343 ordinary-push suspicions the classifier called `vulnerable`, only 40 sit on a line the
|
||||
push changed and only 93 within 10 lines of one. The hunter searches the whole changed file, so on
|
||||
an ordinary push it re-reports the repo's pre-existing code. Scoping to the change is the biggest
|
||||
lever, confidence the second, the `config_secrets` family the third. In-sample on 12 positives, so
|
||||
treat the table as direction, not a tuned threshold:
|
||||
|
||||
| Filter on hunter suspicions | Recall | False alarms per 100 |
|
||||
|---|---|---|
|
||||
| raw | 6/12 | 92.5 |
|
||||
| classifier `vulnerable` | 5/12 | 54.4 |
|
||||
| `vulnerable`, confidence >= 0.8 | 5/12 | 31.8 |
|
||||
| `vulnerable`, within 10 lines of the diff | 5/12 | 22.6 |
|
||||
| `vulnerable`, confidence >= 0.8, within 10 lines | 5/12 | 11.3 |
|
||||
| `vulnerable`, confidence >= 0.8, within 10 lines, not `config_secrets` | 5/12 | 8.8 |
|
||||
|
||||
The exact-line scope (0 lines) drops recall to 1/12 because the benchmark's "head" is the code
|
||||
before the fix, where most fixes add lines rather than change them; a developer's own push has the
|
||||
vulnerable lines in the diff by construction. Proof stays the last step, on what survives this.
|
||||
|
||||
### What to do with this
|
||||
|
||||
1. The push hook needs the LLM tier (owned by `pre-push-safety-net`): the hook as shipped caught
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Sweep candidate precision filters over the pilot's hunter records (no model calls).
|
||||
|
||||
For every hunter defect: classifier verdict, confidence, taxonomy family, and whether its
|
||||
location sits on a line the push changed (git diff base..head, head side). Then score each
|
||||
filter with the real scorer: recall over the 12 positives, false alarms per 100 ordinary pushes.
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, "/home/mc/Source/OpenUltraSAST")
|
||||
from benchmarks.unseen import score # noqa: E402
|
||||
|
||||
P = Path("/home/mc/ousast-results/valid/pilot")
|
||||
repos = json.load(open(P / "repos_hist.json"))
|
||||
rows = score.load_records(P / "hunter")
|
||||
HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
|
||||
|
||||
|
||||
def changed_lines(repo, base, head):
|
||||
out = subprocess.run(
|
||||
["git", "-C", repos[repo], "diff", "-U0", f"{base}..{head}", "--", "."],
|
||||
capture_output=True, text=True, check=False,
|
||||
).stdout
|
||||
lines, path = {}, None
|
||||
for line in out.splitlines():
|
||||
if line.startswith("+++ b/"):
|
||||
path = line[6:]
|
||||
elif line.startswith("@@") and path:
|
||||
m = HUNK.match(line)
|
||||
start, n = int(m.group(1)), int(m.group(2) or 1)
|
||||
lines.setdefault(path, set()).update(range(start, start + max(n, 1)))
|
||||
return lines
|
||||
|
||||
|
||||
def near_changed(defect, lines, slack):
|
||||
for path, line in defect.get("locations", []):
|
||||
hits = lines.get(path)
|
||||
if hits and any(abs(line - h) <= slack for h in hits):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def filtered(row, keep):
|
||||
r = copy.deepcopy(row)
|
||||
adm = r.setdefault("push", {}).setdefault("admission", {})
|
||||
adm["defects"] = [d for d in adm.get("defects", []) if keep(d)]
|
||||
return r
|
||||
|
||||
|
||||
def rate(rows_, keep):
|
||||
pos = [r for r in rows_ if r["kind"] != "ordinary"]
|
||||
ord_ = [r for r in rows_ if r["kind"] == "ordinary" and r.get("status") != "instrument_failure"]
|
||||
caught = sum(bool(score.catches(filtered(r, keep))) for r in pos)
|
||||
alarms = sum(bool(list(score.findings(filtered(r, keep)))) for r in ord_)
|
||||
flagged = alarms + sum(bool(list(score.findings(filtered(r, keep)))) for r in pos)
|
||||
return caught, len(pos), alarms, len(ord_), flagged
|
||||
|
||||
|
||||
# annotate defects with changed-line proximity
|
||||
for r in rows:
|
||||
lines = changed_lines(r["repo"], r["base"], r["head"])
|
||||
for d in r.get("push", {}).get("admission", {}).get("defects", []):
|
||||
d["_changed0"] = near_changed(d, lines, 0)
|
||||
d["_changed10"] = near_changed(d, lines, 10)
|
||||
|
||||
C = lambda d: d.get("classifier") or {} # noqa: E731
|
||||
V = lambda d: C(d).get("verdict") # noqa: E731
|
||||
conf = lambda d: C(d).get("confidence") or 0 # noqa: E731
|
||||
|
||||
filters = {
|
||||
"raw (all suspicions)": lambda d: True,
|
||||
"verdict=vulnerable": lambda d: V(d) == "vulnerable",
|
||||
"vulnerable, conf>=0.7": lambda d: V(d) == "vulnerable" and conf(d) >= 0.7,
|
||||
"vulnerable, conf>=0.8": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8,
|
||||
"vulnerable, conf>=0.85": lambda d: V(d) == "vulnerable" and conf(d) >= 0.85,
|
||||
"vulnerable, not config_secrets": lambda d: V(d) == "vulnerable" and d.get("family") != "config_secrets",
|
||||
"vulnerable, conf>=0.8, not config_secrets": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8 and d.get("family") != "config_secrets",
|
||||
"vulnerable, on a changed line": lambda d: V(d) == "vulnerable" and d["_changed0"],
|
||||
"vulnerable, within 10 of a changed line": lambda d: V(d) == "vulnerable" and d["_changed10"],
|
||||
"vulnerable, conf>=0.7, within 10 of changed": lambda d: V(d) == "vulnerable" and conf(d) >= 0.7 and d["_changed10"],
|
||||
"vulnerable, conf>=0.8, within 10 of changed": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8 and d["_changed10"],
|
||||
"vulnerable, conf>=0.8, within 10, not config_secrets": lambda d: V(d) == "vulnerable" and conf(d) >= 0.8 and d["_changed10"] and d.get("family") != "config_secrets",
|
||||
}
|
||||
print(f"{'filter':58} recall false alarms /100 flagged pushes")
|
||||
for name, keep in filters.items():
|
||||
c, n, a, m, f = rate(rows, keep)
|
||||
print(f"{name:58} {c:2}/{n:<2} {a:3}/{m} = {100*a/m:5.1f} {f}")
|
||||
|
||||
# where do ordinary-push suspicions sit relative to the diff?
|
||||
tot = Counter()
|
||||
for r in rows:
|
||||
if r["kind"] != "ordinary":
|
||||
continue
|
||||
for d in r.get("push", {}).get("admission", {}).get("defects", []):
|
||||
if V(d) == "vulnerable":
|
||||
tot["vulnerable total"] += 1
|
||||
tot["on changed line"] += d["_changed0"]
|
||||
tot["within 10"] += d["_changed10"]
|
||||
tot[f"family {d.get('family')}"] += 1
|
||||
print("\nordinary-push suspicions with verdict=vulnerable:", dict(tot))
|
||||
confs = Counter(round(conf(d), 2) for r in rows if r["kind"] == "ordinary" for d in r.get("push", {}).get("admission", {}).get("defects", []) if V(d) == "vulnerable")
|
||||
print("confidence histogram (ordinary, vulnerable):", dict(sorted(confs.items())))
|
||||
confs = Counter(round(conf(d), 2) for r in rows if r["kind"] != "ordinary" for d in r.get("push", {}).get("admission", {}).get("defects", []) if V(d) == "vulnerable" and score.catches(filtered(r, lambda x, d=d: x is d)))
|
||||
print("confidence of the defects that score a catch:", dict(sorted(confs.items())))
|
||||
Reference in New Issue
Block a user