diff --git a/benchmarks/measurements/2026-10-11-unseen-07-slice1-baseline/NOTES.md b/benchmarks/measurements/2026-10-11-unseen-07-slice1-baseline/NOTES.md new file mode 100644 index 0000000..5678a11 --- /dev/null +++ b/benchmarks/measurements/2026-10-11-unseen-07-slice1-baseline/NOTES.md @@ -0,0 +1,25 @@ +# Record 07 — slice-1 unseen-repo baseline (G0 exit) + +Measured 2026-10-11. Commit: e716e3beab8b01f63ac59dd49586b5904231e2a4 +Image: ghcr.io/norandom/ousast-replay-task@sha256:a24fd03c6ccac0852d495e8a1931e630fffc866ca9e82dd81011533da413f783 +Pool: benchmarks/unseen/pool-p1.toml (100 frozen never-seen repos, freeze_digest 3371c67d…), slice 1, 1300 changes. +Ledger: benchmarks/unseen/usage-p1.jsonl — one `baseline` row (this is G0's exit). + +## Headline (DEFAULT arm, docker lane) +- catch (recall): 3/92 = 3.3% (Wilson95 1.1%–9.2%) +- false alarm: 3.1% (cluster95 1.6%–5.4%, icc 0.25 — repo clustering real) +- alerts_only: 0/1154 = 0% (no hard alerts; catches are advisory/quick stream) +- coverage: quick_only 1246, engine_covered 0, unanalysable 0, instrument_failure 54 (4.2%, < 5% guard) +- by family (catch): deserialization 1/4, injection 2/20, all others 0 +- unseen gate: not_run (no paired production arm; baseline only) + +## Caveats (read before citing) +- **Machine-label**: labels are SZZ-derived from fix commits, 100 unchecked by a human. Catch/FP are vs machine labels. +- **Regex-only floor**: model="disabled", max_model_calls=0, cost_usd=0 — NO LLM; and engine_covered=0 — the Joern + engine never completed within the 30s default budget (the ~200s fixed-overhead wall, see + .kiro/specs/engine-change-scoped-analysis/brief.md). So this is the quick pattern-rule tier ALONE, the floor, + not the product. It is NOT the ≥90%/<10% gate and must not be read as release-grade detection. +- **Docker lane, local** (not the AX/kube lane; the AX completion path is operator-owned). +- **Sizing**: 100 vulns is ~±10% for one overall catch number; per-family (deser=4, config=10) is far too thin for + family-level decisions — needs slices 2+. +- Raw per-change records streamed to the RustFS store; local subdirs pruned-as-we-go (records + inputs.json kept). diff --git a/benchmarks/measurements/2026-10-11-unseen-07-slice1-baseline/record.json b/benchmarks/measurements/2026-10-11-unseen-07-slice1-baseline/record.json new file mode 100644 index 0000000..831fba9 --- /dev/null +++ b/benchmarks/measurements/2026-10-11-unseen-07-slice1-baseline/record.json @@ -0,0 +1,704 @@ +{ + "alerts_only": { + "count": 0, + "n": 1154, + "rate": 0.0, + "wilson95": [ + 0, + 0.003317897715887907 + ] + }, + "breakdowns": { + "coverage": { + "engine_covered": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "quick_only": { + "alerts_only": { + "count": 0, + "n": 1154, + "rate": 0.0, + "wilson95": [ + 0, + 0.003317897715887907 + ] + }, + "catch": { + "count": 3, + "n": 92, + "rate": 0.03260869565217391, + "wilson95": [ + 0.011150975342623229, + 0.09153512338694722 + ] + }, + "false_alarm": { + "cluster95": [ + 0.015638575152041704, + 0.05426356589147287 + ], + "count": 36, + "icc": 0.2454308804673944, + "n": 1154, + "rate": 0.03119584055459272, + "wilson95": [ + 0.022617425634090606, + 0.042885143974740164 + ] + } + }, + "unanalysable": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "family": { + "access_control": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 16, + "rate": 0.0, + "wilson95": [ + 0, + 0.19361341827271994 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "config_secrets": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 10, + "rate": 0.0, + "wilson95": [ + 0, + 0.2775401687666166 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "deserialization": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 1, + "n": 4, + "rate": 0.25, + "wilson95": [ + 0.045586062644636216, + 0.6993639475573634 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "injection": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 2, + "n": 20, + "rate": 0.1, + "wilson95": [ + 0.027865893984153206, + 0.30103820641179335 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "output_encoding": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 16, + "rate": 0.0, + "wilson95": [ + 0, + 0.19361341827271994 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "path": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 13, + "rate": 0.0, + "wilson95": [ + 0, + 0.22810184305529166 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "unknown": { + "alerts_only": { + "count": 0, + "n": 1154, + "rate": 0.0, + "wilson95": [ + 0, + 0.003317897715887907 + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + 0.015638575152041704, + 0.05426356589147287 + ], + "count": 36, + "icc": 0.2454308804673944, + "n": 1154, + "rate": 0.03119584055459272, + "wilson95": [ + 0.022617425634090606, + 0.042885143974740164 + ] + } + }, + "untrusted_destination": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 13, + "rate": 0.0, + "wilson95": [ + 0, + 0.22810184305529166 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "method": { + "first_parent": { + "alerts_only": { + "count": 0, + "n": 1154, + "rate": 0.0, + "wilson95": [ + 0, + 0.003317897715887907 + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + 0.015638575152041704, + 0.05426356589147287 + ], + "count": 36, + "icc": 0.2454308804673944, + "n": 1154, + "rate": 0.03119584055459272, + "wilson95": [ + 0.022617425634090606, + 0.042885143974740164 + ] + } + }, + "introducing": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 3, + "n": 81, + "rate": 0.037037037037037035, + "wilson95": [ + 0.012674884458548488, + 0.10332481380268184 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "last_touch": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 11, + "rate": 0.0, + "wilson95": [ + 2.7755575615628914e-17, + 0.2588400172488141 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "post_cutoff": { + "False": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 3, + "n": 75, + "rate": 0.04, + "wilson95": [ + 0.013695816001486477, + 0.11113168623083756 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "True": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 17, + "rate": 0.0, + "wilson95": [ + 0, + 0.18432366037156456 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "unknown": { + "alerts_only": { + "count": 0, + "n": 1154, + "rate": 0.0, + "wilson95": [ + 0, + 0.003317897715887907 + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + 0.015638575152041704, + 0.05426356589147287 + ], + "count": 36, + "icc": 0.2454308804673944, + "n": 1154, + "rate": 0.03119584055459272, + "wilson95": [ + 0.022617425634090606, + 0.042885143974740164 + ] + } + } + }, + "stream": { + "advisory": { + "count": 0, + "n": 92, + "rate": 0.0, + "wilson95": [ + 0, + 0.04008280329209862 + ] + }, + "alerts": { + "count": 0, + "n": 92, + "rate": 0.0, + "wilson95": [ + 0, + 0.04008280329209862 + ] + }, + "quick": { + "count": 3, + "n": 92, + "rate": 0.03260869565217391, + "wilson95": [ + 0.011150975342623229, + 0.09153512338694722 + ] + } + } + }, + "catch": { + "count": 3, + "n": 92, + "rate": 0.03260869565217391, + "wilson95": [ + 0.011150975342623229, + 0.09153512338694722 + ] + }, + "confirmed": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "coverage": { + "engine_covered": 0, + "instrument_failure": 54, + "quick_only": 1246, + "unanalysable": 0 + }, + "deadlines": [ + { + "applied_deadline_scale": 1.0, + "count": 1300, + "deadline_scale": 1.0, + "deadline_seconds": 30, + "lane": "docker" + } + ], + "false_alarm": { + "cluster95": [ + 0.015638575152041704, + 0.05426356589147287 + ], + "count": 36, + "icc": 0.2454308804673944, + "n": 1154, + "rate": 0.03119584055459272, + "wilson95": [ + 0.022617425634090606, + 0.042885143974740164 + ] + }, + "total": 1300, + "unseen": { + "gate": "not_run" + } +} diff --git a/benchmarks/unseen/usage-p1.jsonl b/benchmarks/unseen/usage-p1.jsonl new file mode 100644 index 0000000..e65460a --- /dev/null +++ b/benchmarks/unseen/usage-p1.jsonl @@ -0,0 +1 @@ +{"date": "2026-10-11", "decision": "record07_baseline", "digest": "70a4c58dab80165d976410c952895374581ef92fad62bf47c313cbc4d3a05801", "freeze_digest": "3371c67d5360f868d13a8fc1cac93a093210f2c40baa566db533de194b32cd8a", "previous": "0000000000000000000000000000000000000000000000000000000000000000", "purpose": "baseline", "result_digest": "704ac7551630668eeaaa0fa609bc0209039b31a0d3f1ff29bdd5a0687ec34e68", "slice": 1}