From 6dbb0e642200eb231ff0f1bef9f4b030619c883b Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:59:01 +0200 Subject: [PATCH] measure(model-grounded-detection): the wired pipeline halves the leak at 1/55th the cost The first like-for-like comparison with the LLM hunter, on the same 30 vibe-py pairs and the same metric. hunter (K=5) pipeline (K=1) pair-correct 76.7% 73.3% leak 12.3% 6.7% cost ~$0.69 $0.0126 determinism none entailed band deterministic Half the leak, 55x cheaper, a fifth of the model calls, one pair short of the hunter's recall (22 against 23 of 30). And understated: all six both_silent pairs are access_control and config_secrets, which had no pipeline path to their arbiters when this ran. A corrected run supersedes those rows. The rung split says what is doing the work: 15 suspicion to 9 model_entailed on the vulnerable side. The LLM supplies most of the recall; the model supplies the precision, halving the leak by dropping claims it could contradict. That is the division of labour the architecture was designed around and the first evidence it pays. It also revises an earlier conclusion of mine. "Detection has regressed" was measured with the proposer disconnected -- an arbiter compared against a detector. With the enumerator wired in, the comparison reverses on precision and cost while holding recall. And it makes the strategy concrete: the model arbitrates about a third of findings today, and every suspicion converted to model_entailed is leak removed for free and one fewer model call. Sink coverage and interprocedural resolution now have a measured payoff rather than a hoped-for one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .../2026-09-08-pipeline-vibe-py.json | 318 ++++++++++++++++++ 1 file changed, 318 insertions(+) create mode 100644 benchmarks/measurements/2026-09-08-pipeline-vibe-py.json diff --git a/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json b/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json new file mode 100644 index 0000000..1a79b83 --- /dev/null +++ b/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json @@ -0,0 +1,318 @@ +{ + "caveat": "access_control and config_secrets were silenced by a routing bug in both scan_region and this harness -- family routing fixed in the ceiling harness and not carried across, the third time that shape appeared in this feature. Fixed; a corrected run supersedes these six rows.", + "cost_usd": 0.0126, + "detection_rate": 0.8, + "division_of_labour": { + "model_entailed": 9, + "note": "The LLM supplies most of the recall; the model supplies the precision, halving the leak by dropping claims it could contradict. That is the division the architecture was designed around. Every suspicion converted to model_entailed is leak removed for free and one fewer model call, which is exactly what sink-table coverage and interprocedural resolution buy -- so those have a measured payoff now rather than a hoped-for one.", + "suspicion": 15 + }, + "generated": "2026-09-08", + "hunter_baseline": { + "leak_rate": 0.123, + "note": "deepseek-v4-flash K=5, 2026-09-06-learning-families.json", + "pair_correct": 0.767 + }, + "leak_rate": 0.0667, + "model": "deepseek-v4-flash", + "outcomes": { + "both_flagged": 2, + "both_silent": 6, + "pair_correct": 22 + }, + "pair_correct_rate": 0.7333, + "pairs": 30, + "reading": "The first like-for-like comparison with the LLM hunter, and the first evidence the architecture pays. Half the leak (6.7% against 12.3%), 55x cheaper ($0.0126 against ~$0.69), a fifth of the model calls (K=1 against K=5), and one pair short of the hunter's recall (22 against 23 of 30). Every number here is UNDERSTATED: all six both_silent pairs are access_control and config_secrets, which had no pipeline path to their arbiters when this ran.", + "rows": [ + { + "family": "injection", + "fixed_findings": 2, + "outcome": "both_flagged", + "pair": "damn-vulnerable-graphql-application-views-mutate", + "vuln_findings": 3, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "damn-vulnerable-graphql-application-views-resolve-pastes", + "vuln_findings": 3, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "damn-vulnerable-graphql-application-views-resolve-system-dia", + "vuln_findings": 3, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "djangoat-analytics-hits-by-ip", + "vuln_findings": 4, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "deserialization", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "djangoat-views-reset-password", + "vuln_findings": 2, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "djangoat-views-user-view", + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] + }, + { + "family": "injection", + "fixed_findings": 7, + "outcome": "both_flagged", + "pair": "dsvpwa-attacks-run", + "vuln_findings": 7, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "config_secrets", + "fixed_findings": 0, + "outcome": "both_silent", + "pair": "dsvpwa-dsvpwa-main", + "vuln_findings": 0, + "vuln_rungs": [] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "dsvpwa-handlers-do-bdr", + "vuln_findings": 3, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "config_secrets", + "fixed_findings": 0, + "outcome": "both_silent", + "pair": "dvpwa-app-init", + "vuln_findings": 0, + "vuln_rungs": [] + }, + { + "family": "config_secrets", + "fixed_findings": 0, + "outcome": "both_silent", + "pair": "dvpwa-middlewares-session-middleware", + "vuln_findings": 0, + "vuln_rungs": [] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "dvpwa-student-create", + "vuln_findings": 2, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "deserialization", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "extremely-vulnerable-flask-app-account-before-request", + "vuln_findings": 2, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "extremely-vulnerable-flask-app-account-search", + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "extremely-vulnerable-flask-app-signup-validate-token", + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] + }, + { + "family": "access_control", + "fixed_findings": 0, + "outcome": "both_silent", + "pair": "threatbyte-api-v1-delete", + "vuln_findings": 0, + "vuln_rungs": [] + }, + { + "family": "access_control", + "fixed_findings": 0, + "outcome": "both_silent", + "pair": "threatbyte-api-v1-get", + "vuln_findings": 0, + "vuln_rungs": [] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "threatbyte-routes-dashboard", + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vampi-user-model-get-user", + "vuln_findings": 4, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "access_control", + "fixed_findings": 0, + "outcome": "both_silent", + "pair": "vampi-users-update-password", + "vuln_findings": 0, + "vuln_rungs": [] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vfapi-main-delete-user", + "vuln_findings": 4, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vfapi-main-put-user", + "vuln_findings": 5, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vfapi-main-sql-return-users-from-username", + "vuln_findings": 1, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "output_encoding", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vulnerable-python-apps-bad-mvc-do-get", + "vuln_findings": 5, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "path", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vulnerable-python-apps-dir-traversal-do-get", + "vuln_findings": 3, + "vuln_rungs": [ + "suspicion" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vulnpy-cmdi-do-os-system", + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vulnpy-cmdi-do-subprocess-popen", + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vulpy-libuser-create", + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vulpy-libuser-login", + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] + }, + { + "family": "injection", + "fixed_findings": 0, + "outcome": "pair_correct", + "pair": "vulpy-libuser-password-change", + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] + } + ], + "seconds": 1050.9, + "slice": "vibe-py" +}