From c74e101d106027c1585ed6f5d209327f62cb48a6 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:18:06 +0200 Subject: [PATCH] measure(model-grounded-detection): 86.7% pair-correct at half the hunter's leak hunter (K=5) pipeline (K=1) pair-correct 76.7% 86.7% leak 12.3% 6.7% detection -- 93.3% cost ~$0.69 $0.0067 Ten points of recall over the hunter, half the leak, a fifth of the model calls, ~100x cheaper. 26 of 30 pairs correct. Four runs, and each step came from a measurement contradicting an assumption -- twice one I had stated: 1 73.3% the absence and config families had no pipeline path at all 2 73.3% routing fixed and the result IDENTICAL, which was the clue: the arbiter was reached and its verdict thrown away, because every corroboration was put to the judge as a taint question 3 80.0% residual question phrased for the arbiter that raised it 4 86.7% aiohttp Application(debug=True) and RedisStorage(httponly=False) added to the settings table The division of labour keeps moving the right way: model_entailed 9 -> 11 while suspicion held at 15, so both findings gained in run 4 came from the MODEL, not the LLM, and cost fell from $0.0126 to $0.0067 as work moved onto the free deterministic path. That is the payoff curve for fact-table coverage, measured rather than argued: closing a gap moves findings out of the LLM's column into the model's, improving precision and cost at once. Recorded corrections and limits, because the numbers are only worth what the caveats are. The three silent config pairs were called corpus defects on a similarity threshold borrowed from the OWASP minimal-fix analysis; it does not apply to a slice that deliberately pairs a vulnerable function with a different safe one, where 36% and 34% pairs both score pair_correct. Two were a settings gap and now split. And permissive_values is a flat literal list with no keyword association, holding both True and False -- debug=True is dangerous where httponly=True is safe, and the model cannot tell them apart. It did not bite here, and it is a real limit of the fact schema. Four remaining misses, each with a named cause. Four runs cost about $0.04. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .../2026-09-08-pipeline-vibe-py.json | 75 +++++++++++-------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json b/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json index 70f3302..d7902aa 100644 --- a/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json +++ b/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json @@ -1,11 +1,12 @@ { - "cost_note": "Three full runs of this measurement cost about $0.03 in total.", - "cost_usd": 0.0076, - "detection_rate": 0.8667, + "correction": "The three silent config pairs were reported as corpus defects on a similarity measure (23%, 45%, 53%). That threshold came from the OWASP minimal-fix analysis and does not apply to this slice: vibe-py pairs a vulnerable function with a DIFFERENT, SAFE function from the same repository, so low similarity is the design. djangoat-views-user-view is 36% similar and dsvpwa-handlers-do-bdr 34%, both pair_correct. Two of the three were a settings-table gap and now split.", + "cost_note": "Four full runs of this measurement cost roughly $0.04 in total.", + "cost_usd": 0.0067, + "detection_rate": 0.9333, "division_of_labour": { "model_corroborated": 2, - "model_entailed": 9, - "note": "The model arbitrates 11 of 26 findings. Every conversion from suspicion to a model rung removes a leak and a model call at once, which is the payoff curve for sink coverage and interprocedural resolution: closing them moves findings out of the LLM's column into the model's.", + "model_entailed": 11, + "note": "The model arbitrates 13 of 28 findings, up from 9 of 24 in run 1. Both findings gained in run 4 came from the MODEL, not the LLM, and cost fell from $0.0126 to $0.0067 as work moved onto the free deterministic path. That is the payoff curve for fact-table coverage, measured: closing a gap moves findings out of the LLM's column into the model's, improving precision and cost at once.", "suspicion": 15 }, "generated": "2026-09-08", @@ -14,16 +15,26 @@ "note": "deepseek-v4-flash K=5, 2026-09-06-learning-families.json", "pair_correct": 0.767 }, + "known_limit": "permissive_values is a flat list of literals with no keyword association, so it holds both True and False. debug=True is dangerous while httponly=True is safe and the model cannot tell them apart. It did not bite here -- run 4's leak is identical to run 3's -- but it is a real limit of the fact schema.", "leak_rate": 0.0667, "model": "deepseek-v4-flash", "outcomes": { "both_flagged": 2, - "both_silent": 4, - "pair_correct": 24 + "both_silent": 2, + "pair_correct": 26 }, - "pair_correct_rate": 0.8, + "pair_correct_rate": 0.8667, "pairs": 30, - "reading": "The wired pipeline beats the LLM hunter on both axes: 80.0% pair-correct against 76.7%, and 6.7% leak against 12.3% -- at one fifth of the model calls (K=1 against K=5) and ~90x cheaper ($0.0076 against ~$0.69). This is the first like-for-like comparison; the earlier 'detection has regressed' reading was taken with the proposer disconnected (an arbiter judged against a detector) and before four wiring bugs were found.", + "reading": "The wired pipeline beats the LLM hunter on both axes: 86.7% pair-correct against 76.7%, and 6.7% leak against 12.3% -- at one fifth of the model calls (K=1 against K=5) and ~100x cheaper ($0.0067 against ~$0.69). The earlier 'detection has regressed' reading was taken with the proposer disconnected, an arbiter judged against a detector, and before four wiring bugs were found.", + "remaining_misses": { + "both_flagged": { + "cause": "the leak: LLM proposals the model cannot contradict", + "family": "injection", + "pairs": 2 + }, + "dsvpwa-dsvpwa-main": "its setting is an argparse default (--ssl defaulting to 0), a CLI default rather than a security-setting call. Predicted before the settings change and unchanged by it.", + "vampi-users-update-password": "the dominance arbiter returns corroborated for it in isolation; still undiagnosed at pipeline level." + }, "rows": [ { "family": "injection", @@ -40,7 +51,7 @@ "fixed_findings": 0, "outcome": "pair_correct", "pair": "damn-vulnerable-graphql-application-views-resolve-pastes", - "vuln_findings": 4, + "vuln_findings": 3, "vuln_rungs": [ "suspicion" ] @@ -108,7 +119,7 @@ "fixed_findings": 0, "outcome": "pair_correct", "pair": "dsvpwa-handlers-do-bdr", - "vuln_findings": 2, + "vuln_findings": 4, "vuln_rungs": [ "suspicion" ] @@ -116,18 +127,22 @@ { "family": "config_secrets", "fixed_findings": 0, - "outcome": "both_silent", + "outcome": "pair_correct", "pair": "dvpwa-app-init", - "vuln_findings": 0, - "vuln_rungs": [] + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] }, { "family": "config_secrets", "fixed_findings": 0, - "outcome": "both_silent", + "outcome": "pair_correct", "pair": "dvpwa-middlewares-session-middleware", - "vuln_findings": 0, - "vuln_rungs": [] + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] }, { "family": "injection", @@ -232,7 +247,7 @@ "fixed_findings": 0, "outcome": "pair_correct", "pair": "vfapi-main-put-user", - "vuln_findings": 4, + "vuln_findings": 5, "vuln_rungs": [ "suspicion" ] @@ -252,7 +267,7 @@ "fixed_findings": 0, "outcome": "pair_correct", "pair": "vulnerable-python-apps-bad-mvc-do-get", - "vuln_findings": 4, + "vuln_findings": 5, "vuln_rungs": [ "suspicion" ] @@ -333,21 +348,17 @@ }, { "leak": 0.067, - "note": "residual question phrased for the arbiter that raised it; 2 of 3 access_control pairs converted from silent to pair-correct", + "note": "residual question phrased for the arbiter that raised it; 2 of 3 access_control pairs converted", "pair_correct": 0.8, "run": 3 + }, + { + "leak": 0.067, + "note": "aiohttp Application/RedisStorage added to the settings table; 2 more config pairs converted, leak unchanged", + "pair_correct": 0.867, + "run": 4 } ], - "seconds": 1071.1, - "slice": "vibe-py", - "still_silent": { - "access_control": { - "cause": "vampi-users-update-password: the dominance arbiter returns corroborated for it in isolation, so this is a pipeline-level issue still to diagnose.", - "pairs": 1 - }, - "config_secrets": { - "cause": "the settings table is Flask/FastAPI-shaped (CORS, set_cookie, app.run); these pairs are aiohttp (session_middleware, setup_redis). Known since group 4.", - "pairs": 3 - } - } + "seconds": 1029.0, + "slice": "vibe-py" }