mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5
fe45081b43
CI was red on `ruff format --check`: 23 files. I ran `ruff check` after every
change this session and never once ran `ruff format --check`, though CI runs
both. Lint passing is not formatting passing, and I treated one as evidence for
the other for the whole session.
Formatting applied. The full CI sequence now reproduces locally: format, lint,
mypy over 94 files, 735 tests, three gates byte-identical, compileall, and the
semantic job's 16 marked tests.
Also records run 5 of the pipeline measurement, which the vampi fix lifted as
predicted:
hunter (K=5) pipeline (K=1)
pair-correct 76.7% 90.0%
leak 12.3% 6.7%
detection -- 96.7%
cost ~$0.69 $0.0068
27 of 30 pairs. Five runs, each step from a measurement contradicting an
assumption: 73.3 (no pipeline path) -> 73.3 (routing fixed, IDENTICAL, which
exposed the discarded verdict) -> 80.0 (residual question matched to its
arbiter) -> 86.7 (aiohttp settings) -> 90.0 (dominance question no longer
contradicting its own evidence).
One miss left: dsvpwa-dsvpwa-main, whose setting is an argparse --ssl default
rather than a security-setting call. Out of reach before the settings work and
still is; contorting the arbiter around one pair would be worse than the miss.
The caveat is recorded beside the number: 30 pairs of one slice, and the slice
iterated against all session. The +16% overfitting gap against vfc-js is why
contributor-scan Req 8.1/8.2 exist.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
352 lines
9.0 KiB
JSON
352 lines
9.0 KiB
JSON
{
|
|
"caveat": "30 pairs of ONE slice, and the slice iterated against all session. The +16% overfitting gap against vfc-js is why contributor-scan Req 8.1 and 8.2 exist. A contributor-facing tool has to hold up on a repository nobody tuned against, which is what that spec's Req 4 and 10 are for.",
|
|
"cost_usd": 0.0068,
|
|
"detection_rate": 0.9667,
|
|
"generated": "2026-09-08",
|
|
"hunter_baseline": {
|
|
"leak_rate": 0.123,
|
|
"note": "deepseek-v4-flash K=5, 2026-09-06-learning-families.json",
|
|
"pair_correct": 0.767
|
|
},
|
|
"leak_rate": 0.0667,
|
|
"model": "deepseek-v4-flash",
|
|
"outcomes": {
|
|
"both_flagged": 2,
|
|
"both_silent": 1,
|
|
"pair_correct": 27
|
|
},
|
|
"pair_correct_rate": 0.9,
|
|
"pairs": 30,
|
|
"reading": "The wired pipeline beats the LLM hunter on both axes: 90.0% pair-correct against 76.7%, and 6.7% leak against 12.3% -- at one fifth of the model calls (K=1 against K=5) and ~100x cheaper ($0.0068 against ~$0.69). 27 of 30 pairs correct, one miss remaining. The earlier 'detection has regressed' reading was taken with the proposer disconnected -- an arbiter judged against a detector -- and before five wiring bugs were found.",
|
|
"remaining_miss": "dsvpwa-dsvpwa-main: its setting is an argparse default (--ssl defaulting to 0), a CLI default rather than a security-setting call. Predicted out of reach before the settings work and still is; contorting the arbiter around one pair would be worse than the miss.",
|
|
"rows": [
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 2,
|
|
"outcome": "both_flagged",
|
|
"pair": "damn-vulnerable-graphql-application-views-mutate",
|
|
"vuln_findings": 3,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "damn-vulnerable-graphql-application-views-resolve-pastes",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "damn-vulnerable-graphql-application-views-resolve-system-dia",
|
|
"vuln_findings": 3,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "djangoat-analytics-hits-by-ip",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "deserialization",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "djangoat-views-reset-password",
|
|
"vuln_findings": 2,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "djangoat-views-user-view",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 7,
|
|
"outcome": "both_flagged",
|
|
"pair": "dsvpwa-attacks-run",
|
|
"vuln_findings": 7,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "config_secrets",
|
|
"fixed_findings": 0,
|
|
"outcome": "both_silent",
|
|
"pair": "dsvpwa-dsvpwa-main",
|
|
"vuln_findings": 0,
|
|
"vuln_rungs": []
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "dsvpwa-handlers-do-bdr",
|
|
"vuln_findings": 3,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "config_secrets",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "dvpwa-app-init",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "config_secrets",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "dvpwa-middlewares-session-middleware",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "dvpwa-student-create",
|
|
"vuln_findings": 2,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "deserialization",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "extremely-vulnerable-flask-app-account-before-request",
|
|
"vuln_findings": 2,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "extremely-vulnerable-flask-app-account-search",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "extremely-vulnerable-flask-app-signup-validate-token",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "access_control",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "threatbyte-api-v1-delete",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_corroborated"
|
|
]
|
|
},
|
|
{
|
|
"family": "access_control",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "threatbyte-api-v1-get",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_corroborated"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "threatbyte-routes-dashboard",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vampi-user-model-get-user",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "access_control",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vampi-users-update-password",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_corroborated"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vfapi-main-delete-user",
|
|
"vuln_findings": 4,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vfapi-main-put-user",
|
|
"vuln_findings": 5,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vfapi-main-sql-return-users-from-username",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "output_encoding",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulnerable-python-apps-bad-mvc-do-get",
|
|
"vuln_findings": 5,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "path",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulnerable-python-apps-dir-traversal-do-get",
|
|
"vuln_findings": 3,
|
|
"vuln_rungs": [
|
|
"suspicion"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulnpy-cmdi-do-os-system",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulnpy-cmdi-do-subprocess-popen",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulpy-libuser-create",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulpy-libuser-login",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
},
|
|
{
|
|
"family": "injection",
|
|
"fixed_findings": 0,
|
|
"outcome": "pair_correct",
|
|
"pair": "vulpy-libuser-password-change",
|
|
"vuln_findings": 1,
|
|
"vuln_rungs": [
|
|
"model_entailed"
|
|
]
|
|
}
|
|
],
|
|
"run_history": [
|
|
{
|
|
"note": "absence and config families had no pipeline path to their arbiters",
|
|
"pair_correct": 0.733,
|
|
"run": 1
|
|
},
|
|
{
|
|
"note": "routing fixed -- IDENTICAL result, which was the clue: the arbiter was reached and its verdict discarded",
|
|
"pair_correct": 0.733,
|
|
"run": 2
|
|
},
|
|
{
|
|
"note": "residual question phrased for the arbiter that raised it",
|
|
"pair_correct": 0.8,
|
|
"run": 3
|
|
},
|
|
{
|
|
"note": "aiohttp Application(debug=True) and RedisStorage(httponly=False) added to the settings table",
|
|
"pair_correct": 0.867,
|
|
"run": 4
|
|
},
|
|
{
|
|
"note": "the dominance question no longer contradicts its own evidence line",
|
|
"pair_correct": 0.9,
|
|
"run": 5
|
|
}
|
|
],
|
|
"seconds": 1213.5,
|
|
"slice": "vibe-py"
|
|
}
|