From 0b1d41d01b3a135626df1e19e4df178325341892 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:43:59 +0100 Subject: [PATCH] measure(judge) gate 2: zero-shot judge over prover findings --- .kiro/steering/roadmap.md | 5 + .../2026-10-28-access-control-judge/NOTES.md | 60 +++++ .../2026-10-28-access-control-judge/run.json | 9 + .../summary.json | 21 ++ benchmarks/valid/judge_pass.py | 232 ++++++++++++++++++ 5 files changed, 327 insertions(+) create mode 100644 benchmarks/measurements/2026-10-28-access-control-judge/NOTES.md create mode 100644 benchmarks/measurements/2026-10-28-access-control-judge/run.json create mode 100644 benchmarks/measurements/2026-10-28-access-control-judge/summary.json create mode 100644 benchmarks/valid/judge_pass.py diff --git a/.kiro/steering/roadmap.md b/.kiro/steering/roadmap.md index c4ae009..73715ad 100644 --- a/.kiro/steering/roadmap.md +++ b/.kiro/steering/roadmap.md @@ -29,6 +29,11 @@ starts until the gates below are read. entry points), 6 covered-but-unflagged for gate 3. Gate 1 holds; proceed to gate 2. 2. Zero-shot judge pass over those traces ("is this entry meant to be public"): does it drop false alarms without killing recall? The pool reports how often the judge is right (correction, rule 3). + - **Read 2026-10-28** (`benchmarks/measurements/2026-10-28-access-control-judge/`): 340 judgments over + all prover findings, 0 failures. Keeps 15/15 true flags (recall 65.2% unchanged), drops only 5/69 + false pushes (FA 15.0 -> 13.9/100). The public-entry question does not separate the prover's false + alarms; the wrongness lives in the trace, not the entry's publicity. Judge is safe but nearly inert; + the FA lever is trace/fact coverage -> gate 3. Do not tune this judge prompt. 3. Trace-component ablation (the amended `hunter-learning-loop` role -- selection, not fitting): which trace components the judge needs to see, with intervals. 4. One pilot run under the at-most-once rule (`hunter-learning-loop` req 8.6): trace + judge over the 12 diff --git a/benchmarks/measurements/2026-10-28-access-control-judge/NOTES.md b/benchmarks/measurements/2026-10-28-access-control-judge/NOTES.md new file mode 100644 index 0000000..2a3a9c7 --- /dev/null +++ b/benchmarks/measurements/2026-10-28-access-control-judge/NOTES.md @@ -0,0 +1,60 @@ +# Zero-shot judge over prover traces -- training pool (gate 2) + +Date: 2026-10-28. Roadmap: `.kiro/steering/roadmap.md`, "measure the trace first", gate 2. +Input: gate 1 prover findings (`~/ousast-results/prover/ac-train-1/`, measurements in +`../2026-10-28-access-control-prover/`). + +## Setup + +- Judge: `benchmarks/valid/judge_pass.py` (run + summarize). One zero-shot judgment per + prover finding (340 findings on 84 flagged pushes: 15 true + 69 false). +- Question, fixed before any answer was seen: "is this entry meant to be public?" with a + strict JSON answer `{public: bool, confidence, reason}` and the statement that + `public: true` means the flag is a false alarm. Payload per judgment: repository, entry + metadata, the embedded trace, evidence lines, operation, missing obligation. No + examples, no tuning -- the pool measures, never trains (roadmap rule 3). +- Model: `openai/deepseek-flash`, temperature 0, per-run dspy cache. 340/340 judgments, + **0 failures**, 34.4 min (~7 s per judgment), no provider errors (the hunt arm's + `LMInvalidRequestError` mode did not appear). +- Rollup rule: a push is dropped only when ALL its findings are judged public. + Raw judgments: `~/ousast-results/prover/ac-judge-1/judgments.jsonl`; summary and run + metadata copied here. + +## Instrument notes + +Two checks before believing the numbers. First, a 2-judgment dry run on one vuln and one +ordinary finding: parse worked, reasons were real. Second, the harness fired a +"run complete" notification while the process was still writing (340 claimed, 319-333 on +disk); a summarize in that window produced partial numbers and was discarded. Final +numbers below are from the completed run only: 340 lines, 340 unique keys, run.json +`judged: 340`, `failures: 0`. + +## Result + +Headline (from `summary.json`): judge over prover findings: keeps 15/15 true flags, drops +5/69 false flags (recall 65.2%, FA 13.9/100). + +| metric | before judge (gate 1) | after judge | +|---|---|---| +| recall (fix pushes) | 15/23 = 65.2% | 15/23 = 65.2% (no true flag lost) | +| false alarms per 100 pushes | 15.0 (69/460) | 13.9 (64/460) | +| precision (push level) | 15/84 = 17.9% | 15/79 = 19.0% | + +Finding-level verdicts (340 judgments): on the 15 true pushes the judge voted +`protected` 90 times and `public` once -- it almost never argues with the prover on a +real vulnerability. On the 69 false pushes it voted `protected` 216 times and `public` +33 times: it calls only about 14% of the false-alarm findings "meant to be public". + +## Reading + +Lane 2 is safe but nearly inert on this pool. It costs no recall, but the public-entry +question does not separate the prover's false alarms: 64 of 69 false pushes survive +because at least one of their findings looks genuinely protected (internal endpoints, +admin-adjacent code -- exactly the code the prover was built to flag). The judge cannot +see why the flag is wrong; the wrongness lives in the trace (an unchecked operation that +is in fact guarded by a mechanism the trace does not model, or an entry that is not +attacker-reachable), which is lane 1 territory. + +Decision input for gate 5: do not spend effort tuning this judge prompt. The FA lever is +trace and fact coverage -- measure it in gate 3 (ablation over the 8 prover misses and a +sample of the 64 judge-kept false pushes) before considering any judge changes. diff --git a/benchmarks/measurements/2026-10-28-access-control-judge/run.json b/benchmarks/measurements/2026-10-28-access-control-judge/run.json new file mode 100644 index 0000000..ba7aaf3 --- /dev/null +++ b/benchmarks/measurements/2026-10-28-access-control-judge/run.json @@ -0,0 +1,9 @@ +{ + "model": "openai/deepseek-flash", + "temperature": 0.0, + "dspy_version": "3.4.1", + "judged": 340, + "failures": 0, + "seconds": 2064.3, + "note": "zero-shot: no examples, no thresholds; question fixed before any answer was seen" +} diff --git a/benchmarks/measurements/2026-10-28-access-control-judge/summary.json b/benchmarks/measurements/2026-10-28-access-control-judge/summary.json new file mode 100644 index 0000000..c92cdc3 --- /dev/null +++ b/benchmarks/measurements/2026-10-28-access-control-judge/summary.json @@ -0,0 +1,21 @@ +{ + "model": "openai/deepseek-flash", + "judgments": 340, + "failed": 0, + "pushes_flagged_by_prover": 84, + "true_pushes_kept": "15/15", + "false_pushes_dropped": "5/69", + "recall_after_judge": 0.6522, + "fa_per_100_after_judge": 13.9, + "finding_verdicts": { + "on_vulnerability_pushes": { + "protected": 90, + "public": 1 + }, + "on_ordinary_pushes": { + "protected": 216, + "public": 33 + } + }, + "headline": "Judge over prover findings: keeps 15/15 true flags, drops 5/69 false flags (recall 65.2%, FA 13.9/100)" +} diff --git a/benchmarks/valid/judge_pass.py b/benchmarks/valid/judge_pass.py new file mode 100644 index 0000000..c66890d --- /dev/null +++ b/benchmarks/valid/judge_pass.py @@ -0,0 +1,232 @@ +"""Gate 2: a zero-shot judge pass over the prover's findings. + +The prover flags an entry when it cannot show a guard dominating a protected +operation. Some flagged entries are meant to be public -- a health endpoint, a +login form, a public API -- and the trace cannot tell. The judge is asked +exactly that question, zero-shot, and this module measures how often the +answer is right against the pool's labels. + +Pool measures, never trains: no example in the prompt, no threshold tuned on +these answers. Every judgment is written to disk before anything is summed, +and the summary refuses to report from an empty or mostly-failed instrument. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path + +JUDGE_QUESTION = ( + "A static analyzer flagged this code path as a possible access-control " + "vulnerability: an attacker-reachable entry point reaches a protected " + "operation without a guard that dominates it.\n\n" + "The analyzer cannot decide one thing: whether the entry point is MEANT TO " + "BE PUBLIC -- intentionally callable by anyone, with no authorization " + "expected -- in which case the missing guard is by design and the flag is a " + "false alarm.\n\n" + "You are given the repository name, the analyzer's finding, and the " + "ordered trace it found (entry, call hops, operation, guards seen, missing " + "obligations). Judge only from what is shown.\n\n" + "Answer with strict JSON, nothing else:\n" + '{"public": true|false, "confidence": "high"|"medium"|"low", "reason": ""}\n\n' + '"public": true means the entry is meant to be callable by anyone and the ' + "flag is a false alarm. public false means the entry should be protected " + "and the flag stands." +) + + +def _finding_key(record_id: str, finding_id: str) -> str: + return hashlib.sha256(f"{record_id}\n{finding_id}".encode()).hexdigest() + + +def _judge_input(repo: str, finding: dict) -> str: + payload = { + "repository": repo, + "entry": finding.get("entry", {}), + "trace": finding.get("trace") or {}, + "evidence": finding.get("evidence", []), + "operation": finding.get("operation", {}), + "missing": finding.get("missing"), + } + return JUDGE_QUESTION + "\n\n" + json.dumps(payload, indent=2, default=str) + + +def _iter_findings(prover_out: Path): + """Every finding of every completed prover record, with its push context.""" + for path in sorted(prover_out.glob("[0-9a-f]*.json")): + record = json.loads(path.read_text()) + if not record.get("done"): + continue + for finding in record.get("findings") or []: + yield record, finding + + +def _load_done(out: Path) -> dict: + done = {} + judged = out / "judgments.jsonl" + if judged.is_file(): + for line in judged.read_text().splitlines(): + if line.strip(): + row = json.loads(line) + done[row["key"]] = row + return done + + +def run(args) -> int: + import dspy + + from openultrasast.hunt.program import build_lm + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + lm = build_lm(cache_dir=out / "dspy-cache") + dspy.configure(lm=lm, adapter=dspy.ChatAdapter()) + + class Judge(dspy.Signature): + """Answer the question with strict JSON only.""" + + case: str = dspy.InputField() + answer: str = dspy.OutputField() + + judge = dspy.Predict(Judge) + done = _load_done(out) + judgments_path = out / "judgments.jsonl" + + todo = 0 + failures = [] + started = time.monotonic() + for record, finding in _iter_findings(Path(args.prover_out)): + key = _finding_key(record["id"], finding.get("id") or "") + if key in done: + continue + todo += 1 + payload = { + "key": key, + "record_id": record["id"], + "repo": record["repo"], + "kind": record.get("kind"), + "finding_id": finding.get("id"), + "path": finding.get("path"), + "line": finding.get("line"), + "model": lm.model, + } + try: + result = judge(case=_judge_input(record["repo"], finding)) + text = (result.answer or "").strip() + # Tolerate a fenced JSON block, then parse strictly. + body = text.strip("`") + if body.lower().startswith("json"): + body = body[4:] + parsed = json.loads(body[body.index("{") : body.rindex("}") + 1]) + if not isinstance(parsed.get("public"), bool): + raise ValueError("public is not a boolean") + payload["public"] = parsed["public"] + payload["confidence"] = parsed.get("confidence") + payload["reason"] = parsed.get("reason") + payload["status"] = "ok" + except Exception as error: # noqa: BLE001 (recorded, never swallowed silently) + payload["status"] = "judge_error" + payload["error_type"] = type(error).__name__ + payload["error"] = str(error)[:500] + failures.append(payload) + with judgments_path.open("a") as handle: + handle.write(json.dumps(payload, default=str) + "\n") + if not todo % 25: + print(f"{todo} judged, {len(failures)} failures", flush=True) + + seconds = time.monotonic() - started + meta = { + "model": lm.model, + "temperature": 0.0, + "dspy_version": dspy.__version__, + "judged": todo, + "failures": len(failures), + "seconds": round(seconds, 1), + "note": "zero-shot: no examples, no thresholds; question fixed before any answer was seen", + } + (out / "run.json").write_text(json.dumps(meta, indent=2, default=str) + "\n") + print(json.dumps(meta)) + return 2 if failures and todo else 0 + + +def summarize(args) -> int: + out = Path(args.out) + done = _load_done(out) + rows = list(done.values()) + if not rows: + raise SystemExit("no judgments: refuse to summarize an empty instrument") + + failures = [r for r in rows if r.get("status") != "ok"] + if len(failures) > 0.1 * len(rows): + raise SystemExit(f"refuse to summarize: {len(failures)}/{len(rows)} judgments failed") + + # Push-level rollup: a flagged push is dropped only when every one of its + # findings was judged public. + pushes = {} + for row in rows: + slot = pushes.setdefault(row["record_id"], {"repo": row["repo"], "kind": row["kind"], "kept": 0, "dropped": 0}) + if row["public"]: + slot["dropped"] += 1 + else: + slot["kept"] += 1 + + def rate(n, d): + return round(n / d, 4) if d else None + + vuln = [p for p in pushes.values() if p["kind"] == "vulnerability"] + ordinary = [p for p in pushes.values() if p["kind"] == "ordinary"] + tp_kept = sum(1 for p in vuln if p["kept"]) + fa_total, fa_dropped = len(ordinary), sum(1 for p in ordinary if not p["kept"]) + + summary = { + "model": rows[0].get("model"), + "judgments": len(rows), + "failed": len(failures), + "pushes_flagged_by_prover": len(pushes), + "true_pushes_kept": f"{tp_kept}/{len(vuln)}", + "false_pushes_dropped": f"{fa_dropped}/{fa_total}", + "recall_after_judge": rate(tp_kept, 23), + "fa_per_100_after_judge": round((fa_total - fa_dropped) / 460 * 100, 1), + "finding_verdicts": { + "on_vulnerability_pushes": { + "protected": sum(1 for r in rows if r["kind"] == "vulnerability" and not r["public"]), + "public": sum(1 for r in rows if r["kind"] == "vulnerability" and r["public"]), + }, + "on_ordinary_pushes": { + "protected": sum(1 for r in rows if r["kind"] == "ordinary" and not r["public"]), + "public": sum(1 for r in rows if r["kind"] == "ordinary" and r["public"]), + }, + }, + "headline": ( + f"Judge over prover findings: keeps {tp_kept}/{len(vuln)} true flags, " + f"drops {fa_dropped}/{fa_total} false flags " + f"(recall {rate(tp_kept, 23)*100:.1f}%, FA {(fa_total - fa_dropped) / 460 * 100:.1f}/100)" + ), + } + (out / "summary.json").write_text(json.dumps(summary, indent=2, default=str) + "\n") + print(json.dumps(summary, indent=2, default=str)) + return 0 + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + + p_run = sub.add_parser("run", help="judge every finding in a prover out-dir") + p_run.add_argument("prover_out", type=Path) + p_run.add_argument("--out", type=Path, required=True) + p_run.set_defaults(fn=run) + + p_sum = sub.add_parser("summarize", help="roll judgments up to push-level metrics") + p_sum.add_argument("--out", type=Path, required=True) + p_sum.set_defaults(fn=summarize) + + args = parser.parse_args() + raise SystemExit(args.fn(args)) + + +if __name__ == "__main__": + main()