diff --git a/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json b/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json index 70f3302..d7902aa 100644 --- a/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json +++ b/benchmarks/measurements/2026-09-08-pipeline-vibe-py.json @@ -1,11 +1,12 @@ { - "cost_note": "Three full runs of this measurement cost about $0.03 in total.", - "cost_usd": 0.0076, - "detection_rate": 0.8667, + "correction": "The three silent config pairs were reported as corpus defects on a similarity measure (23%, 45%, 53%). That threshold came from the OWASP minimal-fix analysis and does not apply to this slice: vibe-py pairs a vulnerable function with a DIFFERENT, SAFE function from the same repository, so low similarity is the design. djangoat-views-user-view is 36% similar and dsvpwa-handlers-do-bdr 34%, both pair_correct. Two of the three were a settings-table gap and now split.", + "cost_note": "Four full runs of this measurement cost roughly $0.04 in total.", + "cost_usd": 0.0067, + "detection_rate": 0.9333, "division_of_labour": { "model_corroborated": 2, - "model_entailed": 9, - "note": "The model arbitrates 11 of 26 findings. Every conversion from suspicion to a model rung removes a leak and a model call at once, which is the payoff curve for sink coverage and interprocedural resolution: closing them moves findings out of the LLM's column into the model's.", + "model_entailed": 11, + "note": "The model arbitrates 13 of 28 findings, up from 9 of 24 in run 1. Both findings gained in run 4 came from the MODEL, not the LLM, and cost fell from $0.0126 to $0.0067 as work moved onto the free deterministic path. That is the payoff curve for fact-table coverage, measured: closing a gap moves findings out of the LLM's column into the model's, improving precision and cost at once.", "suspicion": 15 }, "generated": "2026-09-08", @@ -14,16 +15,26 @@ "note": "deepseek-v4-flash K=5, 2026-09-06-learning-families.json", "pair_correct": 0.767 }, + "known_limit": "permissive_values is a flat list of literals with no keyword association, so it holds both True and False. debug=True is dangerous while httponly=True is safe and the model cannot tell them apart. It did not bite here -- run 4's leak is identical to run 3's -- but it is a real limit of the fact schema.", "leak_rate": 0.0667, "model": "deepseek-v4-flash", "outcomes": { "both_flagged": 2, - "both_silent": 4, - "pair_correct": 24 + "both_silent": 2, + "pair_correct": 26 }, - "pair_correct_rate": 0.8, + "pair_correct_rate": 0.8667, "pairs": 30, - "reading": "The wired pipeline beats the LLM hunter on both axes: 80.0% pair-correct against 76.7%, and 6.7% leak against 12.3% -- at one fifth of the model calls (K=1 against K=5) and ~90x cheaper ($0.0076 against ~$0.69). This is the first like-for-like comparison; the earlier 'detection has regressed' reading was taken with the proposer disconnected (an arbiter judged against a detector) and before four wiring bugs were found.", + "reading": "The wired pipeline beats the LLM hunter on both axes: 86.7% pair-correct against 76.7%, and 6.7% leak against 12.3% -- at one fifth of the model calls (K=1 against K=5) and ~100x cheaper ($0.0067 against ~$0.69). The earlier 'detection has regressed' reading was taken with the proposer disconnected, an arbiter judged against a detector, and before four wiring bugs were found.", + "remaining_misses": { + "both_flagged": { + "cause": "the leak: LLM proposals the model cannot contradict", + "family": "injection", + "pairs": 2 + }, + "dsvpwa-dsvpwa-main": "its setting is an argparse default (--ssl defaulting to 0), a CLI default rather than a security-setting call. Predicted before the settings change and unchanged by it.", + "vampi-users-update-password": "the dominance arbiter returns corroborated for it in isolation; still undiagnosed at pipeline level." + }, "rows": [ { "family": "injection", @@ -40,7 +51,7 @@ "fixed_findings": 0, "outcome": "pair_correct", "pair": "damn-vulnerable-graphql-application-views-resolve-pastes", - "vuln_findings": 4, + "vuln_findings": 3, "vuln_rungs": [ "suspicion" ] @@ -108,7 +119,7 @@ "fixed_findings": 0, "outcome": "pair_correct", "pair": "dsvpwa-handlers-do-bdr", - "vuln_findings": 2, + "vuln_findings": 4, "vuln_rungs": [ "suspicion" ] @@ -116,18 +127,22 @@ { "family": "config_secrets", "fixed_findings": 0, - "outcome": "both_silent", + "outcome": "pair_correct", "pair": "dvpwa-app-init", - "vuln_findings": 0, - "vuln_rungs": [] + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] }, { "family": "config_secrets", "fixed_findings": 0, - "outcome": "both_silent", + "outcome": "pair_correct", "pair": "dvpwa-middlewares-session-middleware", - "vuln_findings": 0, - "vuln_rungs": [] + "vuln_findings": 1, + "vuln_rungs": [ + "model_entailed" + ] }, { "family": "injection", @@ -232,7 +247,7 @@ "fixed_findings": 0, "outcome": "pair_correct", "pair": "vfapi-main-put-user", - "vuln_findings": 4, + "vuln_findings": 5, "vuln_rungs": [ "suspicion" ] @@ -252,7 +267,7 @@ "fixed_findings": 0, "outcome": "pair_correct", "pair": "vulnerable-python-apps-bad-mvc-do-get", - "vuln_findings": 4, + "vuln_findings": 5, "vuln_rungs": [ "suspicion" ] @@ -333,21 +348,17 @@ }, { "leak": 0.067, - "note": "residual question phrased for the arbiter that raised it; 2 of 3 access_control pairs converted from silent to pair-correct", + "note": "residual question phrased for the arbiter that raised it; 2 of 3 access_control pairs converted", "pair_correct": 0.8, "run": 3 + }, + { + "leak": 0.067, + "note": "aiohttp Application/RedisStorage added to the settings table; 2 more config pairs converted, leak unchanged", + "pair_correct": 0.867, + "run": 4 } ], - "seconds": 1071.1, - "slice": "vibe-py", - "still_silent": { - "access_control": { - "cause": "vampi-users-update-password: the dominance arbiter returns corroborated for it in isolation, so this is a pipeline-level issue still to diagnose.", - "pairs": 1 - }, - "config_secrets": { - "cause": "the settings table is Flask/FastAPI-shaped (CORS, set_cookie, app.run); these pairs are aiohttp (session_middleware, setup_redis). Known since group 4.", - "pairs": 3 - } - } + "seconds": 1029.0, + "slice": "vibe-py" }