mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
measure(g0): record 07 — slice-1 unseen-repo baseline (G0 exit)
First honest, measured unseen-repo baseline on 100 frozen never-seen repos (1300 changes, DEFAULT arm, docker lane). This is the quick pattern-rule FLOOR (no LLM: model=disabled, 0 calls; engine_covered=0, the ~200s wall) — the release gate is now a measurement, not an assertion. - catch 3/92 = 3.3% (Wilson95 1.1–9.2%); false_alarm 3.1% (cluster95 1.6–5.4%, icc 0.25); alerts_only 0%; instrument_failure 4.2% (<5%); engine_covered 0. - by family: deser 1/4, injection 2/20, rest 0. - baseline row appended to usage-p1.jsonl (freeze_digest 3371c67d) = G0 exit. Caveats in NOTES.md: machine-label (SZZ, 100 unchecked), regex-only floor (NOT the >=90%/<10% gate), docker lane, slice-1 sizing (~±10% overall, per-family too thin). Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
e716e3beab
commit
e901640f3f
@@ -0,0 +1,25 @@
|
||||
# Record 07 — slice-1 unseen-repo baseline (G0 exit)
|
||||
|
||||
Measured 2026-10-11. Commit: e716e3beab8b01f63ac59dd49586b5904231e2a4
|
||||
Image: ghcr.io/norandom/ousast-replay-task@sha256:a24fd03c6ccac0852d495e8a1931e630fffc866ca9e82dd81011533da413f783
|
||||
Pool: benchmarks/unseen/pool-p1.toml (100 frozen never-seen repos, freeze_digest 3371c67d…), slice 1, 1300 changes.
|
||||
Ledger: benchmarks/unseen/usage-p1.jsonl — one `baseline` row (this is G0's exit).
|
||||
|
||||
## Headline (DEFAULT arm, docker lane)
|
||||
- catch (recall): 3/92 = 3.3% (Wilson95 1.1%–9.2%)
|
||||
- false alarm: 3.1% (cluster95 1.6%–5.4%, icc 0.25 — repo clustering real)
|
||||
- alerts_only: 0/1154 = 0% (no hard alerts; catches are advisory/quick stream)
|
||||
- coverage: quick_only 1246, engine_covered 0, unanalysable 0, instrument_failure 54 (4.2%, < 5% guard)
|
||||
- by family (catch): deserialization 1/4, injection 2/20, all others 0
|
||||
- unseen gate: not_run (no paired production arm; baseline only)
|
||||
|
||||
## Caveats (read before citing)
|
||||
- **Machine-label**: labels are SZZ-derived from fix commits, 100 unchecked by a human. Catch/FP are vs machine labels.
|
||||
- **Regex-only floor**: model="disabled", max_model_calls=0, cost_usd=0 — NO LLM; and engine_covered=0 — the Joern
|
||||
engine never completed within the 30s default budget (the ~200s fixed-overhead wall, see
|
||||
.kiro/specs/engine-change-scoped-analysis/brief.md). So this is the quick pattern-rule tier ALONE, the floor,
|
||||
not the product. It is NOT the ≥90%/<10% gate and must not be read as release-grade detection.
|
||||
- **Docker lane, local** (not the AX/kube lane; the AX completion path is operator-owned).
|
||||
- **Sizing**: 100 vulns is ~±10% for one overall catch number; per-family (deser=4, config=10) is far too thin for
|
||||
family-level decisions — needs slices 2+.
|
||||
- Raw per-change records streamed to the RustFS store; local subdirs pruned-as-we-go (records + inputs.json kept).
|
||||
@@ -0,0 +1,704 @@
|
||||
{
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 1154,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.003317897715887907
|
||||
]
|
||||
},
|
||||
"breakdowns": {
|
||||
"coverage": {
|
||||
"engine_covered": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"quick_only": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 1154,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.003317897715887907
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 3,
|
||||
"n": 92,
|
||||
"rate": 0.03260869565217391,
|
||||
"wilson95": [
|
||||
0.011150975342623229,
|
||||
0.09153512338694722
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
0.015638575152041704,
|
||||
0.05426356589147287
|
||||
],
|
||||
"count": 36,
|
||||
"icc": 0.2454308804673944,
|
||||
"n": 1154,
|
||||
"rate": 0.03119584055459272,
|
||||
"wilson95": [
|
||||
0.022617425634090606,
|
||||
0.042885143974740164
|
||||
]
|
||||
}
|
||||
},
|
||||
"unanalysable": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"family": {
|
||||
"access_control": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 16,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.19361341827271994
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"config_secrets": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 10,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.2775401687666166
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"deserialization": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 1,
|
||||
"n": 4,
|
||||
"rate": 0.25,
|
||||
"wilson95": [
|
||||
0.045586062644636216,
|
||||
0.6993639475573634
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"injection": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 2,
|
||||
"n": 20,
|
||||
"rate": 0.1,
|
||||
"wilson95": [
|
||||
0.027865893984153206,
|
||||
0.30103820641179335
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"output_encoding": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 16,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.19361341827271994
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"path": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 13,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.22810184305529166
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"unknown": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 1154,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.003317897715887907
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
0.015638575152041704,
|
||||
0.05426356589147287
|
||||
],
|
||||
"count": 36,
|
||||
"icc": 0.2454308804673944,
|
||||
"n": 1154,
|
||||
"rate": 0.03119584055459272,
|
||||
"wilson95": [
|
||||
0.022617425634090606,
|
||||
0.042885143974740164
|
||||
]
|
||||
}
|
||||
},
|
||||
"untrusted_destination": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 13,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.22810184305529166
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"method": {
|
||||
"first_parent": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 1154,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.003317897715887907
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
0.015638575152041704,
|
||||
0.05426356589147287
|
||||
],
|
||||
"count": 36,
|
||||
"icc": 0.2454308804673944,
|
||||
"n": 1154,
|
||||
"rate": 0.03119584055459272,
|
||||
"wilson95": [
|
||||
0.022617425634090606,
|
||||
0.042885143974740164
|
||||
]
|
||||
}
|
||||
},
|
||||
"introducing": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 3,
|
||||
"n": 81,
|
||||
"rate": 0.037037037037037035,
|
||||
"wilson95": [
|
||||
0.012674884458548488,
|
||||
0.10332481380268184
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"last_touch": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 11,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
2.7755575615628914e-17,
|
||||
0.2588400172488141
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"post_cutoff": {
|
||||
"False": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 3,
|
||||
"n": 75,
|
||||
"rate": 0.04,
|
||||
"wilson95": [
|
||||
0.013695816001486477,
|
||||
0.11113168623083756
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"True": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 17,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.18432366037156456
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"unknown": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 1154,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.003317897715887907
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
0.015638575152041704,
|
||||
0.05426356589147287
|
||||
],
|
||||
"count": 36,
|
||||
"icc": 0.2454308804673944,
|
||||
"n": 1154,
|
||||
"rate": 0.03119584055459272,
|
||||
"wilson95": [
|
||||
0.022617425634090606,
|
||||
0.042885143974740164
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"stream": {
|
||||
"advisory": {
|
||||
"count": 0,
|
||||
"n": 92,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.04008280329209862
|
||||
]
|
||||
},
|
||||
"alerts": {
|
||||
"count": 0,
|
||||
"n": 92,
|
||||
"rate": 0.0,
|
||||
"wilson95": [
|
||||
0,
|
||||
0.04008280329209862
|
||||
]
|
||||
},
|
||||
"quick": {
|
||||
"count": 3,
|
||||
"n": 92,
|
||||
"rate": 0.03260869565217391,
|
||||
"wilson95": [
|
||||
0.011150975342623229,
|
||||
0.09153512338694722
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"catch": {
|
||||
"count": 3,
|
||||
"n": 92,
|
||||
"rate": 0.03260869565217391,
|
||||
"wilson95": [
|
||||
0.011150975342623229,
|
||||
0.09153512338694722
|
||||
]
|
||||
},
|
||||
"confirmed": {
|
||||
"alerts_only": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"catch": {
|
||||
"count": 0,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
null,
|
||||
null
|
||||
],
|
||||
"count": 0,
|
||||
"icc": null,
|
||||
"n": 0,
|
||||
"rate": null,
|
||||
"wilson95": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
}
|
||||
},
|
||||
"coverage": {
|
||||
"engine_covered": 0,
|
||||
"instrument_failure": 54,
|
||||
"quick_only": 1246,
|
||||
"unanalysable": 0
|
||||
},
|
||||
"deadlines": [
|
||||
{
|
||||
"applied_deadline_scale": 1.0,
|
||||
"count": 1300,
|
||||
"deadline_scale": 1.0,
|
||||
"deadline_seconds": 30,
|
||||
"lane": "docker"
|
||||
}
|
||||
],
|
||||
"false_alarm": {
|
||||
"cluster95": [
|
||||
0.015638575152041704,
|
||||
0.05426356589147287
|
||||
],
|
||||
"count": 36,
|
||||
"icc": 0.2454308804673944,
|
||||
"n": 1154,
|
||||
"rate": 0.03119584055459272,
|
||||
"wilson95": [
|
||||
0.022617425634090606,
|
||||
0.042885143974740164
|
||||
]
|
||||
},
|
||||
"total": 1300,
|
||||
"unseen": {
|
||||
"gate": "not_run"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"date": "2026-10-11", "decision": "record07_baseline", "digest": "70a4c58dab80165d976410c952895374581ef92fad62bf47c313cbc4d3a05801", "freeze_digest": "3371c67d5360f868d13a8fc1cac93a093210f2c40baa566db533de194b32cd8a", "previous": "0000000000000000000000000000000000000000000000000000000000000000", "purpose": "baseline", "result_digest": "704ac7551630668eeaaa0fa609bc0209039b31a0d3f1ff29bdd5a0687ec34e68", "slice": 1}
|
||||
Reference in New Issue
Block a user