mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(constrained-detector): the ceiling, measured offline across every slice
Group 1 complete. Tasks 1.2 and 1.3: `candidates.ceiling`, `ousast learning
candidates`, and the stop-or-go reading. No model is called anywhere in this
group, so the condition that could have ended the feature was reachable for
free.
Verdict: go.
agent-vfc (deciding) 89.7% ceiling against 44.4% round-zero detection
vfc (deciding) 84.7%, baseline still running, non-gating
vfc-js (deciding) 52.9% -- see below
github (deciding) 100.0%
vibe-py (development)100.0% against 83.3%
The margin is largest on agent-vfc, which is the slice the tool is actually
for.
The measurement forced a shape correction before the detector existed.
config_secrets first read 63.6% with a bind-and-literal shape, because a
permissive default is as often an argument to a call -- cors({origin: '*'}),
app.use(session({secure: false})) -- as it is a binding. Adding call sites to
that family's shape took it to 100% and recovered four agent-vfc rows.
Carry-forward, and not this feature's to fix: vfc-js sits at 52.9% because
eight of seventeen rows label a function the JavaScript CST names `<anon>` --
`module.exports.publish = function () {}` and its relatives. That is a
function-naming gap, it caps the existing hunter on that slice exactly as
much, and it is the same class as the `_declarator_start` defects found in
learning-harness task 4.1.
The ruleset arm of `ceiling` is kept so the 12.1% that rejected it stays a
measurement rather than a remembered number.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5
Claude Opus 5
parent
8d771901bf
commit
78b4f33a8e
@@ -0,0 +1,381 @@
|
||||
{
|
||||
"command": "uv run ousast learning candidates --slice all",
|
||||
"deciding_slices": {
|
||||
"agent-vfc": {
|
||||
"ceiling": 0.897,
|
||||
"headroom": 0.453,
|
||||
"round_zero_detection": 0.444,
|
||||
"verdict": "go"
|
||||
},
|
||||
"github": {
|
||||
"ceiling": 1.0,
|
||||
"round_zero_detection": null,
|
||||
"verdict": "go"
|
||||
},
|
||||
"vfc": {
|
||||
"ceiling": 0.847,
|
||||
"note": "baseline still running; non-gating",
|
||||
"round_zero_detection": null,
|
||||
"verdict": "go"
|
||||
},
|
||||
"vfc-js": {
|
||||
"ceiling": 0.529,
|
||||
"note": "eight of seventeen rows label a function the JavaScript CST names `<anon>`: `module.exports.x = function () {}` and friends. The bug is in function naming, not in candidate generation, and it caps this slice for the existing hunter too.",
|
||||
"round_zero_detection": null,
|
||||
"verdict": "blocked_by_a_naming_gap"
|
||||
}
|
||||
},
|
||||
"development_slice": {
|
||||
"vibe-py": {
|
||||
"ceiling": 1.0,
|
||||
"headroom": 0.167,
|
||||
"round_zero_detection": 0.833
|
||||
}
|
||||
},
|
||||
"gaps": [
|
||||
[
|
||||
"agent-vfc",
|
||||
"edutrace-page-ee7ba2",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"agent-vfc",
|
||||
"lensops-kubernetesmanagement-fe4910",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"agent-vfc",
|
||||
"money-track-category-icons-4d4a5b",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"chromium-cve-2019-5786",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"chromium-cve-2022-2162",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2020-8284",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2020-8286",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2021-22890",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2021-22901-ossl_connect_step1",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2022-27775",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2022-27776",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2022-27778",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2022-27782-curl_init_userdefined",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2022-27782-nss_setup_connect",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2022-27782-ossl_connect_step1",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2023-38545",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"curl-cve-2023-38545-do_socks5",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2021-3449-init_sig_algs_cert",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2021-3712-ia5ncasecmp",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2022-0778",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2022-1292",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2022-2068-hash_dir",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2022-3358",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2022-3358-evp_cipher_init_internal",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2023-0216",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2023-5363",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2024-2511",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2025-69421",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2026-14456-expect_quic_for_value",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc",
|
||||
"openssl-cve-2026-31790-rsasve_generate",
|
||||
"unsupported_language"
|
||||
],
|
||||
[
|
||||
"vfc-js",
|
||||
"apex-publish-static-files-2-0-0",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc-js",
|
||||
"bestzip-2-1-6",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc-js",
|
||||
"clamscan-1-2-0",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc-js",
|
||||
"hot-formula-parser-3-0-0",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc-js",
|
||||
"pdf-image-1-0-5",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc-js",
|
||||
"pidusage-1-0-0",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc-js",
|
||||
"portprocesses-1-0-4",
|
||||
"no_candidate"
|
||||
],
|
||||
[
|
||||
"vfc-js",
|
||||
"serve-here-js-1-1-0",
|
||||
"no_candidate"
|
||||
]
|
||||
],
|
||||
"generated": "2026-09-06",
|
||||
"generator": "ir",
|
||||
"note": "constrained-detector task 1.3, the stop-or-go reading. A candidate-driven detector can only judge sites the enumerator produces, so this is the limit on its recall, measured offline before any model was called. The condition for continuing: on every deciding slice the ceiling must exceed that slice's recorded detection rate from round zero. It does, with the largest margin exactly where it matters most.",
|
||||
"per_family": {
|
||||
"access_control": {
|
||||
"ceiling": 0.9,
|
||||
"pairs": 10,
|
||||
"sites": {
|
||||
"max": 19,
|
||||
"median": 12,
|
||||
"min": 4,
|
||||
"p90": 19
|
||||
},
|
||||
"with_candidate": 9
|
||||
},
|
||||
"config_secrets": {
|
||||
"ceiling": 1.0,
|
||||
"pairs": 11,
|
||||
"sites": {
|
||||
"max": 60,
|
||||
"median": 15,
|
||||
"min": 2,
|
||||
"p90": 40
|
||||
},
|
||||
"with_candidate": 11
|
||||
},
|
||||
"deserialization": {
|
||||
"ceiling": 1.0,
|
||||
"pairs": 4,
|
||||
"sites": {
|
||||
"max": 10,
|
||||
"median": 7,
|
||||
"min": 3,
|
||||
"p90": 10
|
||||
},
|
||||
"with_candidate": 4
|
||||
},
|
||||
"injection": {
|
||||
"ceiling": 0.8461538461538461,
|
||||
"pairs": 52,
|
||||
"sites": {
|
||||
"max": 77,
|
||||
"median": 5,
|
||||
"min": 1,
|
||||
"p90": 17
|
||||
},
|
||||
"with_candidate": 44
|
||||
},
|
||||
"memory": {
|
||||
"ceiling": 0.8636363636363636,
|
||||
"pairs": 176,
|
||||
"sites": {
|
||||
"max": 77,
|
||||
"median": 10,
|
||||
"min": 1,
|
||||
"p90": 29
|
||||
},
|
||||
"with_candidate": 152
|
||||
},
|
||||
"output_encoding": {
|
||||
"ceiling": 1.0,
|
||||
"pairs": 2,
|
||||
"sites": {
|
||||
"max": 17,
|
||||
"median": 17,
|
||||
"min": 3,
|
||||
"p90": 17
|
||||
},
|
||||
"with_candidate": 2
|
||||
},
|
||||
"path": {
|
||||
"ceiling": 0.875,
|
||||
"pairs": 8,
|
||||
"sites": {
|
||||
"max": 12,
|
||||
"median": 7,
|
||||
"min": 5,
|
||||
"p90": 12
|
||||
},
|
||||
"with_candidate": 7
|
||||
},
|
||||
"prototype": {
|
||||
"ceiling": 0.7142857142857143,
|
||||
"pairs": 7,
|
||||
"sites": {
|
||||
"max": 14,
|
||||
"median": 6,
|
||||
"min": 6,
|
||||
"p90": 14
|
||||
},
|
||||
"with_candidate": 5
|
||||
},
|
||||
"unknown": {
|
||||
"ceiling": 0.0,
|
||||
"pairs": 2,
|
||||
"sites": {
|
||||
"max": 0,
|
||||
"median": 0,
|
||||
"min": 0,
|
||||
"p90": 0
|
||||
},
|
||||
"with_candidate": 0
|
||||
},
|
||||
"untrusted_destination": {
|
||||
"ceiling": 1.0,
|
||||
"pairs": 5,
|
||||
"sites": {
|
||||
"max": 17,
|
||||
"median": 7,
|
||||
"min": 3,
|
||||
"p90": 17
|
||||
},
|
||||
"with_candidate": 5
|
||||
}
|
||||
},
|
||||
"per_slice": {
|
||||
"agent-vfc": {
|
||||
"ceiling": 0.896551724137931,
|
||||
"pairs": 29,
|
||||
"with_candidate": 26
|
||||
},
|
||||
"github": {
|
||||
"ceiling": 1.0,
|
||||
"pairs": 6,
|
||||
"with_candidate": 6
|
||||
},
|
||||
"local": {
|
||||
"ceiling": 1.0,
|
||||
"pairs": 3,
|
||||
"with_candidate": 3
|
||||
},
|
||||
"sast": {
|
||||
"ceiling": 1.0,
|
||||
"pairs": 11,
|
||||
"with_candidate": 11
|
||||
},
|
||||
"vfc": {
|
||||
"ceiling": 0.8465909090909091,
|
||||
"pairs": 176,
|
||||
"with_candidate": 149
|
||||
},
|
||||
"vfc-js": {
|
||||
"ceiling": 0.5294117647058824,
|
||||
"pairs": 17,
|
||||
"with_candidate": 9
|
||||
},
|
||||
"vibe-py": {
|
||||
"ceiling": 1.0,
|
||||
"pairs": 35,
|
||||
"with_candidate": 35
|
||||
}
|
||||
},
|
||||
"shape_correction": "config_secrets first measured 63.6% with a bind-and-literal shape. A permissive default is as often an argument to a call -- cors({origin: '*'}) -- as a binding, and adding call sites to that family's shape took it to 100%. Found by this measurement, before the detector existed.",
|
||||
"verdict": "go"
|
||||
}
|
||||
Reference in New Issue
Block a user