mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5
4e7fe65f17
hunter (K=5) pipeline (K=1)
pair-correct 76.7% 80.0%
leak 12.3% 6.7%
cost ~$0.69 $0.0076
determinism none entailed band deterministic
Higher recall, half the leak, a fifth of the model calls, ~90x cheaper.
Three runs, and the middle one earned its keep by changing nothing. Run 2 fixed
family routing and returned numbers IDENTICAL to run 1 -- which was the clue
that the dominance arbiter was being reached and its verdict thrown away. Every
corroboration was put to the judge as a taint question ("a sanitizer appears on
the path -- does it suffice?"), meaningless for a missing guard, so the judge
said no and the finding was dropped. Run 3 phrases the residual question for
the arbiter that raised it and converts 2 of 3 access_control pairs from silent
to pair-correct, arriving at model_corroborated -- a rung that existed but had
never reached a finding.
This settles a question open since the start of the session, and reverses an
earlier answer of mine. "Detection has regressed" was measured with the
proposer disconnected: an arbiter compared against a detector, before four
wiring bugs were found.
Division of labour: 15 suspicion, 9 model_entailed, 2 model_corroborated. The
model arbitrates 11 of 26 findings, and every conversion from suspicion to a
model rung removes a leak and a model call at once -- the payoff curve for sink
coverage and interprocedural resolution, now measured rather than assumed.
Still silent: 3 config_secrets (the settings table is Flask/FastAPI-shaped
while the pairs are aiohttp) and 1 access_control still to diagnose.
Three full runs cost about $0.03 in total.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
354 lines
9.1 KiB
JSON
354 lines
9.1 KiB
JSON
{
|
|
"cost_note": "Three full runs of this measurement cost about $0.03 in total.",
|
|
"cost_usd": 0.0076,
|
|
"detection_rate": 0.8667,
|
|
"division_of_labour": {
|
|
"model_corroborated": 2,
|
|
"model_entailed": 9,
|
|
"note": "The model arbitrates 11 of 26 findings. Every conversion from suspicion to a model rung removes a leak and a model call at once, which is the payoff curve for sink coverage and interprocedural resolution: closing them moves findings out of the LLM's column into the model's.",
|
|
"suspicion": 15
|
|
},
|
|
"generated": "2026-09-08",
|
|
"hunter_baseline": {
|
|
"leak_rate": 0.123,
|
|
"note": "deepseek-v4-flash K=5, 2026-09-06-learning-families.json",
|
|
"pair_correct": 0.767
|
|
},
|
|
"leak_rate": 0.0667,
|
|
"model": "deepseek-v4-flash",
|
|
"outcomes": {
|
|
"both_flagged": 2,
|
|
"both_silent": 4,
|
|
"pair_correct": 24
|
|
},
|
|
"pair_correct_rate": 0.8,
|
|
"pairs": 30,
|
|
"reading": "The wired pipeline beats the LLM hunter on both axes: 80.0% pair-correct against 76.7%, and 6.7% leak against 12.3% -- at one fifth of the model calls (K=1 against K=5) and ~90x cheaper ($0.0076 against ~$0.69). This is the first like-for-like comparison; the earlier 'detection has regressed' reading was taken with the proposer disconnected (an arbiter judged against a detector) and before four wiring bugs were found.",
|
|
"rows": [
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 2,
|
|
"outcome": "both_flagged",
|
|
"pair": "damn-vulnerable-graphql-application-views-mutate",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "damn-vulnerable-graphql-application-views-resolve-pastes",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "damn-vulnerable-graphql-application-views-resolve-system-dia",
|
|
"vuln_findings": 3,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "djangoat-analytics-hits-by-ip",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "deserialization",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "djangoat-views-reset-password",
|
|
"vuln_findings": 2,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "djangoat-views-user-view",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 7,
|
|
"outcome": "both_flagged",
|
|
"pair": "dsvpwa-attacks-run",
|
|
"vuln_findings": 7,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "config_secrets",
|
|
"fixed_findings": 0,
|
|
"outcome": "both_silent",
|
|
"pair": "dsvpwa-dsvpwa-main",
|
|
"vuln_findings": 0,
|
|
"vuln_rungs": []
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "dsvpwa-handlers-do-bdr",
|
|
"vuln_findings": 2,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "config_secrets",
|
|
"fixed_findings": 0,
|
|
"outcome": "both_silent",
|
|
"pair": "dvpwa-app-init",
|
|
"vuln_findings": 0,
|
|
"vuln_rungs": []
|
|
},
|
|
{
|
|
"family": "config_secrets",
|
|
"fixed_findings": 0,
|
|
"outcome": "both_silent",
|
|
"pair": "dvpwa-middlewares-session-middleware",
|
|
"vuln_findings": 0,
|
|
"vuln_rungs": []
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "dvpwa-student-create",
|
|
"vuln_findings": 2,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "deserialization",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "extremely-vulnerable-flask-app-account-before-request",
|
|
"vuln_findings": 2,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "extremely-vulnerable-flask-app-account-search",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "extremely-vulnerable-flask-app-signup-validate-token",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "access_control",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "threatbyte-api-v1-delete",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_corroborated"
|
|
]
|
|
},
|
|
{
|
|
"family": "access_control",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "threatbyte-api-v1-get",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_corroborated"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "threatbyte-routes-dashboard",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vampi-user-model-get-user",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "access_control",
|
|
"fixed_findings": 0,
|
|
"outcome": "both_silent",
|
|
"pair": "vampi-users-update-password",
|
|
"vuln_findings": 0,
|
|
"vuln_rungs": []
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vfapi-main-delete-user",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vfapi-main-put-user",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vfapi-main-sql-return-users-from-username",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "output_encoding",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulnerable-python-apps-bad-mvc-do-get",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "path",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulnerable-python-apps-dir-traversal-do-get",
|
|
"vuln_findings": 3,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulnpy-cmdi-do-os-system",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulnpy-cmdi-do-subprocess-popen",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulpy-libuser-create",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulpy-libuser-login",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulpy-libuser-password-change",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
}
|
|
],
|
|
"run_history": [
|
|
{
|
|
"leak": 0.067,
|
|
"note": "access_control and config_secrets had no pipeline path to their arbiters",
|
|
"pair_correct": 0.733,
|
|
"run": 1
|
|
},
|
|
{
|
|
"leak": 0.067,
|
|
"note": "family routing fixed -- IDENTICAL result, which was the clue: the arbiter was reached and its verdict discarded",
|
|
"pair_correct": 0.733,
|
|
"run": 2
|
|
},
|
|
{
|
|
"leak": 0.067,
|
|
"note": "residual question phrased for the arbiter that raised it; 2 of 3 access_control pairs converted from silent to pair-correct",
|
|
"pair_correct": 0.8,
|
|
"run": 3
|
|
}
|
|
],
|
|
"seconds": 1071.1,
|
|
"slice": "vibe-py",
|
|
"still_silent": {
|
|
"access_control": {
|
|
"cause": "vampi-users-update-password: the dominance arbiter returns corroborated for it in isolation, so this is a pipeline-level issue still to diagnose.",
|
|
"pairs": 1
|
|
},
|
|
"config_secrets": {
|
|
"cause": "the settings table is Flask/FastAPI-shaped (CORS, set_cookie, app.run); these pairs are aiohttp (session_middleware, setup_redis). Known since group 4.",
|
|
"pairs": 3
|
|
}
|
|
}
|
|
}
|