marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

measure(model-grounded-detection): the wired pipeline halves the leak at 1/55th the cost

Browse Source 
The first like-for-like comparison with the LLM hunter, on the same 30 vibe-py
pairs and the same metric.

                       hunter (K=5)   pipeline (K=1)
  pair-correct              76.7%          73.3%
  leak                      12.3%           6.7%
  cost                     ~$0.69        $0.0126
  determinism               none      entailed band deterministic

Half the leak, 55x cheaper, a fifth of the model calls, one pair short of the
hunter's recall (22 against 23 of 30). And understated: all six both_silent
pairs are access_control and config_secrets, which had no pipeline path to
their arbiters when this ran. A corrected run supersedes those rows.

The rung split says what is doing the work: 15 suspicion to 9 model_entailed on
the vulnerable side. The LLM supplies most of the recall; the model supplies
the precision, halving the leak by dropping claims it could contradict. That is
the division of labour the architecture was designed around and the first
evidence it pays.

It also revises an earlier conclusion of mine. "Detection has regressed" was
measured with the proposer disconnected -- an arbiter compared against a
detector. With the enumerator wired in, the comparison reverses on precision
and cost while holding recall.

And it makes the strategy concrete: the model arbitrates about a third of
findings today, and every suspicion converted to model_entailed is leak removed
for free and one fewer model call. Sink coverage and interprocedural resolution
now have a measured payoff rather than a hoped-for one.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-08 17:59:01 +02:00
co-authored by Claude Opus 5
parent 7a76fcb028
commit 6dbb0e6422
1 changed files with 318 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/measurements/2026-09-08-pipeline-vibe-py.json
+318
View File
@@ -0,0 +1,318 @@
{
"caveat": "access_control and config_secrets were silenced by a routing bug in both scan_region and this harness -- family routing fixed in the ceiling harness and not carried across, the third time that shape appeared in this feature. Fixed; a corrected run supersedes these six rows.",
"cost_usd": 0.0126,
"detection_rate": 0.8,
"division_of_labour": {
"model_entailed": 9,
"note": "The LLM supplies most of the recall; the model supplies the precision, halving the leak by dropping claims it could contradict. That is the division the architecture was designed around. Every suspicion converted to model_entailed is leak removed for free and one fewer model call, which is exactly what sink-table coverage and interprocedural resolution buy -- so those have a measured payoff now rather than a hoped-for one.",
"suspicion": 15
},
"generated": "2026-09-08",
"hunter_baseline": {
"leak_rate": 0.123,
"note": "deepseek-v4-flash K=5, 2026-09-06-learning-families.json",
"pair_correct": 0.767
},
"leak_rate": 0.0667,
"model": "deepseek-v4-flash",
"outcomes": {
"both_flagged": 2,
"both_silent": 6,
"pair_correct": 22
},
"pair_correct_rate": 0.7333,
"pairs": 30,
"reading": "The first like-for-like comparison with the LLM hunter, and the first evidence the architecture pays. Half the leak (6.7% against 12.3%), 55x cheaper ($0.0126 against ~$0.69), a fifth of the model calls (K=1 against K=5), and one pair short of the hunter's recall (22 against 23 of 30). Every number here is UNDERSTATED: all six both_silent pairs are access_control and config_secrets, which had no pipeline path to their arbiters when this ran.",
"rows": [
{
"family": "injection",
"fixed_findings": 2,
"outcome": "both_flagged",
"pair": "damn-vulnerable-graphql-application-views-mutate",
"vuln_findings": 3,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "damn-vulnerable-graphql-application-views-resolve-pastes",
"vuln_findings": 3,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "damn-vulnerable-graphql-application-views-resolve-system-dia",
"vuln_findings": 3,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "djangoat-analytics-hits-by-ip",
"vuln_findings": 4,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "deserialization",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "djangoat-views-reset-password",
"vuln_findings": 2,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "djangoat-views-user-view",
"vuln_findings": 1,
"vuln_rungs": [
"model_entailed"
]
},
{
"family": "injection",
"fixed_findings": 7,
"outcome": "both_flagged",
"pair": "dsvpwa-attacks-run",
"vuln_findings": 7,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "config_secrets",
"fixed_findings": 0,
"outcome": "both_silent",
"pair": "dsvpwa-dsvpwa-main",
"vuln_findings": 0,
"vuln_rungs": []
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "dsvpwa-handlers-do-bdr",
"vuln_findings": 3,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "config_secrets",
"fixed_findings": 0,
"outcome": "both_silent",
"pair": "dvpwa-app-init",
"vuln_findings": 0,
"vuln_rungs": []
},
{
"family": "config_secrets",
"fixed_findings": 0,
"outcome": "both_silent",
"pair": "dvpwa-middlewares-session-middleware",
"vuln_findings": 0,
"vuln_rungs": []
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "dvpwa-student-create",
"vuln_findings": 2,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "deserialization",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "extremely-vulnerable-flask-app-account-before-request",
"vuln_findings": 2,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "extremely-vulnerable-flask-app-account-search",
"vuln_findings": 1,
"vuln_rungs": [
"model_entailed"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "extremely-vulnerable-flask-app-signup-validate-token",
"vuln_findings": 1,
"vuln_rungs": [
"model_entailed"
]
},
{
"family": "access_control",
"fixed_findings": 0,
"outcome": "both_silent",
"pair": "threatbyte-api-v1-delete",
"vuln_findings": 0,
"vuln_rungs": []
},
{
"family": "access_control",
"fixed_findings": 0,
"outcome": "both_silent",
"pair": "threatbyte-api-v1-get",
"vuln_findings": 0,
"vuln_rungs": []
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "threatbyte-routes-dashboard",
"vuln_findings": 1,
"vuln_rungs": [
"model_entailed"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vampi-user-model-get-user",
"vuln_findings": 4,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "access_control",
"fixed_findings": 0,
"outcome": "both_silent",
"pair": "vampi-users-update-password",
"vuln_findings": 0,
"vuln_rungs": []
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vfapi-main-delete-user",
"vuln_findings": 4,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vfapi-main-put-user",
"vuln_findings": 5,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vfapi-main-sql-return-users-from-username",
"vuln_findings": 1,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "output_encoding",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vulnerable-python-apps-bad-mvc-do-get",
"vuln_findings": 5,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "path",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vulnerable-python-apps-dir-traversal-do-get",
"vuln_findings": 3,
"vuln_rungs": [
"suspicion"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vulnpy-cmdi-do-os-system",
"vuln_findings": 1,
"vuln_rungs": [
"model_entailed"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vulnpy-cmdi-do-subprocess-popen",
"vuln_findings": 1,
"vuln_rungs": [
"model_entailed"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vulpy-libuser-create",
"vuln_findings": 1,
"vuln_rungs": [
"model_entailed"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vulpy-libuser-login",
"vuln_findings": 1,
"vuln_rungs": [
"model_entailed"
]
},
{
"family": "injection",
"fixed_findings": 0,
"outcome": "pair_correct",
"pair": "vulpy-libuser-password-change",
"vuln_findings": 1,
"vuln_rungs": [
"model_entailed"
]
}
],
"seconds": 1050.9,
"slice": "vibe-py"
}