diff --git a/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/NOTES.md b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/NOTES.md index 88bc085..8df67a1 100644 --- a/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/NOTES.md +++ b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/NOTES.md @@ -1,8 +1,93 @@ # Valid detection benchmark, pilot (2026-10-12) -First build of the benchmark defined in `.kiro/specs/valid-detection-benchmark/`. Labels come from -the fix, not SZZ. The frozen manifest is at `benchmarks/valid/pilot/` (`pool-valid.toml`, -`freeze-valid.json`). This directory keeps the provenance. +First build and first run of the benchmark defined in `.kiro/specs/valid-detection-benchmark/`. +Labels come from the fix, not SZZ. The frozen manifest is at `benchmarks/valid/pilot/` +(`pool-valid.toml`, `freeze-valid.json`). This directory keeps the provenance and the numbers. + +## Numbers (read 2026-10-13 morning) + +Two arms over the same 252 changes (12 verified positives, 240 ordinary pushes from the same 12 +repos), as R6 of the requirements asks: + +- **(a) push hook as shipped**: quick rules + engine, 300 s deadline, the Docker replay lane, no + model key in the container. Still running when this was written: 163 of 252 records done (all 12 + positives, 151 ordinary). The final numbers land in `MORNING.txt` outside the repo and will replace + the `*_interim_163.json` files here. +- **(b) hunter arm**: the LLM tool hunter (`scan --mode deep`'s search) run off-cluster over the + changed files at head, every finding classified into the taxonomy by the decision-engine + classifier. Finished: 252 of 252. Reported twice: raw (every hunter suspicion counts) and + filtered (only suspicions the classifier judged `vulnerable`). + +| Metric | (a) push hook, 163 done | (b) hunter, raw | (b') hunter, classifier-filtered | +|---|---|---|---| +| Recall (12 positives) | 0 / 12 = 0% (Wilson 0 to 24.3) | 6 / 12 = 50% (25.4 to 74.6) | 5 / 12 = 41.7% (19.3 to 68.0) | +| False alarms per 100 ordinary pushes | 1 / 149 = 0.7 (0.1 to 3.7) | 221 / 239 = 92.5 (88.4 to 95.2) | 130 / 239 = 54.4 (48.1 to 60.6) | +| Precision (flagged pushes that were positives) | 0 / 1 | 6 / 232 = 2.6% | 5 / 139 = 3.6% | +| Exploit-verified | 0 (no proof run) | 0 (no proof run) | 0 (no proof run) | +| Instrument failures | 2 / 163 | 1 / 252 | 1 / 252 | + +Neither arm meets the gate (recall at least 90%, false alarms under 10 per 100 pushes). The hook as +shipped is quiet and blind on this set. The hunter finds about half and alarms on more than half of +ordinary pushes even after the classifier. The whole-tool number the brief holds the tool to is +(a) plus (b'); since (a) caught nothing, it equals (b') on recall and adds one false alarm. + +Files: `scored_a_interim_163.json`, `metrics_a_interim_163.json`, `scored_b_raw.json`, +`metrics_b_raw.json`, `scored_b_filtered.json`, `metrics_b_filtered.json`. Scorer: +`benchmarks/unseen/score.py` (a catch needs the labelled family and a location inside the labelled +function or within 10 lines of it). Metrics: `benchmarks/valid/metrics.py`. + +### Per positive + +| Repo | Label | (a) hook | (b) raw | (b') filtered | What the hunter did | +|---|---|---|---|---|---| +| bytebase/dbhub | access_control | miss (engine ran) | miss | miss | 8 suspicions in the right files, all `not_vulnerable` or `unsure`, none typed access_control | +| contentful/contentful-mcp-server | untrusted_destination | miss | miss | miss | 2 `vulnerable` hits in the labelled file, typed `path`; family mismatch, not credited | +| fastapi-users/fastapi-users | access_control | miss | miss | miss | 0 suspicions | +| i18next/i18next-http-middleware | output_encoding | miss | hit | miss | hit at `lib/index.js:96`, classifier said `unsure`, dropped by the filter | +| imzbf/md-editor-v3 | output_encoding | miss (engine ran) | hit | hit | 9 `vulnerable` html-injection suspicions at the labelled sites | +| jupyter-lsp/jupyterlab-lsp | path | miss | hit | hit | 3 `vulnerable` path hits in the labelled file | +| matrix-org/matrix-appservice-bridge | access_control | miss (engine ran) | miss | miss | 5 "complexity hotspot" suspicions in the right file, all `not_vulnerable` | +| nvbn/thefuck | path | miss (engine ran) | hit | hit | both labelled functions hit, `vulnerable` 0.85 to 0.9 | +| ondata/ckan-mcp-server | untrusted_destination | miss | hit | hit | 2 `vulnerable` ssrf hits in `http.ts` | +| webpack/webpack-bundle-analyzer | output_encoding | miss | miss | miss | `vulnerable` output_encoding hit in `viewer.js:185`, outside the labelled function | +| zenitram/malla | output_encoding | miss | hit | hit | 7 `vulnerable` dom-xss hits | +| zereight/gitlab-mcp | path | miss | miss | miss | 5 suspicions, none typed path | + +Two misses are scorer strictness rather than blindness: contentful (right file, family typed `path` +instead of `untrusted_destination`) and webpack (right file and family, other function). Keep the +strict scorer; note them. + +### Arm (a) detail + +- Engine finished inside 300 s on 100 of 163 changes (61%); 61 fell back to quick rules only; 2 + instrument failures (`contentful` 9e706114, `matrix` 7018468e, nothing readable at the hotspots). + On the positives: engine ran on 4 of 12, quick only on 8. +- Every record proves the instrument: hook exit 0, bytes read per changed file, base and head + verified. +- The one false alarm: `bytebase/dbhub` 5bd45460, quick rule `js-command-exec` (CWE-78) at + `src/connectors/sqlite/index.ts:132`, "Node child_process command execution needs review". + +### Arm (b) detail + +- 1398 suspicions over 251 pushes: mean 5.6 per ordinary push, median 5, max 23. +- Classifier verdicts: 377 `vulnerable`, 968 `not_vulnerable`, 53 `unsure`, 0 errors. Families among + the `vulnerable` ones: config_secrets 103, injection 74, output_encoding 58, access_control 54, + path 39, untrusted_destination 33, prototype 10, other 6. +- The hunter's own `family:` tag is free text (`sql-parser`, `complexity-hotspot`, `dom-xss`), so + without the classifier every finding scored `unknown`. Product follow-up: constrain the hunter + prompt to the taxonomy (`tool_hunter.py`) so the push path emits taxonomy families natively. +- The instrument failure: `matrix` 7018468e, no readable hotspot (`hook_bytes_read: 0`). + +### What to do with this + +1. The push hook needs the LLM tier (owned by `pre-push-safety-net`): the hook as shipped caught + none of the 12. That is the product gap R6 names. +2. The hunter needs a precision step before it can sit on pre-push: 54 alarms per 100 pushes after + the classifier is far from 10. Candidates: require `vulnerable` with confidence at or above 0.7, + drop `config_secrets` noise in ordinary pushes, restrict to changed lines, or run the proof arm on + the survivors. +3. Twelve positives give intervals 50 points wide. Grow to the ~40 positives `score.py:required_n` + asks for before quoting a single recall number. ## What was built @@ -15,6 +100,8 @@ the fix, not SZZ. The frozen manifest is at `benchmarks/valid/pilot/` (`pool-val | Negatives | 20 ordinary first-parent commits per positive change, drawn from the same repo, fix excluded. 240 in total. | | Held-out guard | 0 dropped against the training-only used set (1777 names: pairs, pointers, harvest, units, sweep). | | Freeze | 252 changes: 12 `vulnerability` + 240 `ordinary`. Digest in `freeze-valid.json`. | +| Replay, arm (a) | `benchmarks/unseen/replay.py --arm long_deadline`, Docker lane, one container at a time on the local host, about 14 records per hour. Started 2026-10-12 19:51. | +| Hunter, arm (b) | `benchmarks/valid/hunter_arm.py run`, positives 20:23 to 20:51, ordinary 20:51 to 22:00. | ## The 12 positives @@ -52,14 +139,25 @@ change. training source (pairs, pointers, harvest, units, sweep). - **Catalog enumeration skips a site it cannot read** (fix renamed or removed the function) instead of aborting the whole run. Verification stays strict. +- **The hook has no LLM detection tier.** `push/assistance.py` only picks witnesses for defects the + engine already admitted. R6 was amended to measure the hook as shipped and the hook plus the + hunter arm; the second is the number the tool is held to. +- **Hunter findings are classified, never relabelled.** The classifier gets the finding's excerpt + and the taxonomy list; the change's own label is never shown to it. The raw hunter family, title + and rationale stay on the defect for audit. +- **Replay output kept every result twice** (top level and `attempt-*/`), 30 MB a record, and the + host hit 1.1 GB free at record 162. Byte-identical `attempt-*/` copies of finished records were + removed; a guard repeats that while the replay runs. Nothing in Docker was touched. ## Not done yet -- **Replay** (whole tool, all tiers on) has not run. 252 changes at the engine's ~200 s fixed cost is - about 14 hours one container at a time on the local host; that scale belongs on the kube `ax` lane, - still blocked on the gVisor checkpoint path. -- **Metrics** (recall, precision, false alarms per 100 pushes, exploit-verified rate) follow the replay. +- Final arm (a) numbers on all 252 changes (replay finishing 2026-10-13 afternoon), then the + combined (a)+(b') score with `hunter_arm combine --keep-verdicts vulnerable`. +- Exploit-verified rate: the proof arm has not run on any of the 26 flagged positive findings. - Growing from 12 to the ~40 positives `score.py:required_n` asks for. +- The 252-change replay at the engine's ~200 s fixed cost is about 17 hours one container at a time + on the local host; that scale belongs on the kube `ax` lane, still blocked on the gVisor + checkpoint path. ## Reproduce @@ -72,4 +170,8 @@ python - <<'PY' # assemble with the training-only used set from benchmarks.valid import assemble ... # see this directory's provenance files; used = build_used(root) minus {measurements, populations} PY +# arm (a): replay.py --arm long_deadline, docker lane, --docker-image-bytes 3608729816, output outside the repo +# arm (b): DEEPSEEK_API_KEY=... python -m benchmarks.valid.hunter_arm run benchmarks/valid/pilot/pool-valid.toml --slice 1 --repos repos_hist.json --out +python -m benchmarks.unseen.score --out scored.json --slice-id 1 +python -m benchmarks.valid.metrics --scored scored.json --out metrics.json ``` diff --git a/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/metrics_a_interim_163.json b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/metrics_a_interim_163.json new file mode 100644 index 0000000..d5a70a3 --- /dev/null +++ b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/metrics_a_interim_163.json @@ -0,0 +1,52 @@ +{ + "exploit_verified": { + "count": 0, + "flagged_positive_pushes": 0, + "flagged_positives": 0, + "n": 0, + "not_yet_proved": [], + "rate": null, + "status": "measured", + "wilson95": [ + null, + null + ] + }, + "false_alarms_per_100_pushes": { + "cluster95": [ + 0.0, + 2.027027027027027 + ], + "count": 1, + "icc": -0.004445973772031825, + "n": 149, + "rate": 0.6711409395973155, + "wilson95": [ + 0.11856820977550406, + 3.703427895996746 + ] + }, + "headline": "On unseen app repos: recall 0.0% (Wilson 0.0-24.3), false alarms 0.7 per 100 pushes, precision 0.0%; exploit-verified 0 of 0.", + "precision": { + "count": 0, + "n": 1, + "rate": 0.0, + "wilson95": [ + 0, + 0.7934567085261071 + ] + }, + "recall": { + "cluster95": [ + 0.0, + 0.0 + ], + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + } +} diff --git a/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/metrics_b_filtered.json b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/metrics_b_filtered.json new file mode 100644 index 0000000..e5c52e1 --- /dev/null +++ b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/metrics_b_filtered.json @@ -0,0 +1,79 @@ +{ + "exploit_verified": { + "count": 0, + "flagged_positive_pushes": 9, + "flagged_positives": 26, + "n": 0, + "not_yet_proved": [ + "contentful-contentful-mcp-server-22e8b2fc09c3b2c4", + "contentful-contentful-mcp-server-df53a40a29ee8d8d", + "i18next-i18next-http-middleware-f3e2a6886951f36a", + "imzbf-md-editor-v3-14658213915dbde5", + "imzbf-md-editor-v3-b9c833a3524a57a9", + "jupyter-lsp-jupyterlab-lsp-59f9a8b80b133f6f", + "nvbn-thefuck-456ad18acb14f1b2", + "nvbn-thefuck-74126de049d52d8c", + "ondata-ckan-mcp-server-01a3f68f84397eb4", + "ondata-ckan-mcp-server-b0b05cdc830b1f8b", + "webpack-webpack-bundle-analyzer-8e07af8a5ee1cc52", + "zenitram-malla-15753fae1999978e", + "zenitram-malla-45807117f798f314", + "zenitram-malla-4c7ce7ae0b9dd0cf", + "zenitram-malla-5b8f3116fefd595b", + "zenitram-malla-72b8d4dc93a566fb", + "zenitram-malla-73c0175a1009ffbf", + "zenitram-malla-825a0f3ce563c373", + "zenitram-malla-858255cb6145e104", + "zenitram-malla-92e437caca55290b", + "zenitram-malla-aeb1d1e3d40815a8", + "zenitram-malla-bf4b3bcd2652952c", + "zenitram-malla-c3ab7f5f4e769cc6", + "zenitram-malla-f3535c03a02fa0e3", + "zereight-gitlab-mcp-2367b459f8c3802b", + "zereight-gitlab-mcp-768c7d5a32ccc95b" + ], + "rate": null, + "status": "not yet proved", + "wilson95": [ + null, + null + ] + }, + "false_alarms_per_100_pushes": { + "cluster95": [ + 47.083333333333336, + 61.66666666666667 + ], + "count": 130, + "icc": 0.022377467632178107, + "n": 239, + "rate": 54.39330543933054, + "wilson95": [ + 48.0589907402735, + 60.58862153044123 + ] + }, + "headline": "On unseen app repos: recall 41.7% (Wilson 19.3-68.0), false alarms 54.4 per 100 pushes, precision 3.6%; exploit-verified 0 of 26 (26 not yet proved).", + "precision": { + "count": 5, + "n": 139, + "rate": 0.03597122302158273, + "wilson95": [ + 0.015460613110063436, + 0.08144112979956511 + ] + }, + "recall": { + "cluster95": [ + 0.16666666666666666, + 0.6666666666666666 + ], + "count": 5, + "n": 12, + "rate": 0.4166666666666667, + "wilson95": [ + 0.19325746190524656, + 0.6804926643446272 + ] + } +} diff --git a/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/metrics_b_raw.json b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/metrics_b_raw.json new file mode 100644 index 0000000..b5b52fa --- /dev/null +++ b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/metrics_b_raw.json @@ -0,0 +1,81 @@ +{ + "exploit_verified": { + "count": 0, + "flagged_positive_pushes": 11, + "flagged_positives": 28, + "n": 0, + "not_yet_proved": [ + "bytebase-dbhub-f2624a2ead8d31ca", + "contentful-contentful-mcp-server-22e8b2fc09c3b2c4", + "contentful-contentful-mcp-server-df53a40a29ee8d8d", + "i18next-i18next-http-middleware-f3e2a6886951f36a", + "imzbf-md-editor-v3-14658213915dbde5", + "imzbf-md-editor-v3-b9c833a3524a57a9", + "jupyter-lsp-jupyterlab-lsp-59f9a8b80b133f6f", + "matrix-org-matrix-appservice-bridge-3a8081647e10c293", + "nvbn-thefuck-456ad18acb14f1b2", + "nvbn-thefuck-74126de049d52d8c", + "ondata-ckan-mcp-server-01a3f68f84397eb4", + "ondata-ckan-mcp-server-b0b05cdc830b1f8b", + "webpack-webpack-bundle-analyzer-8e07af8a5ee1cc52", + "zenitram-malla-15753fae1999978e", + "zenitram-malla-45807117f798f314", + "zenitram-malla-4c7ce7ae0b9dd0cf", + "zenitram-malla-5b8f3116fefd595b", + "zenitram-malla-72b8d4dc93a566fb", + "zenitram-malla-73c0175a1009ffbf", + "zenitram-malla-825a0f3ce563c373", + "zenitram-malla-858255cb6145e104", + "zenitram-malla-92e437caca55290b", + "zenitram-malla-aeb1d1e3d40815a8", + "zenitram-malla-bf4b3bcd2652952c", + "zenitram-malla-c3ab7f5f4e769cc6", + "zenitram-malla-f3535c03a02fa0e3", + "zereight-gitlab-mcp-2367b459f8c3802b", + "zereight-gitlab-mcp-768c7d5a32ccc95b" + ], + "rate": null, + "status": "not yet proved", + "wilson95": [ + null, + null + ] + }, + "false_alarms_per_100_pushes": { + "cluster95": [ + 89.1213389121339, + 95.83333333333334 + ], + "count": 221, + "icc": 0.006754645062417405, + "n": 239, + "rate": 92.46861924686193, + "wilson95": [ + 88.41031716603341, + 95.1832681175424 + ] + }, + "headline": "On unseen app repos: recall 50.0% (Wilson 25.4-74.6), false alarms 92.5 per 100 pushes, precision 2.6%; exploit-verified 0 of 28 (28 not yet proved).", + "precision": { + "count": 6, + "n": 232, + "rate": 0.02586206896551724, + "wilson95": [ + 0.011905368254414139, + 0.05526513940835613 + ] + }, + "recall": { + "cluster95": [ + 0.25, + 0.75 + ], + "count": 6, + "n": 12, + "rate": 0.5, + "wilson95": [ + 0.2537781703934222, + 0.7462218296065778 + ] + } +} diff --git a/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/scored_a_interim_163.json b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/scored_a_interim_163.json new file mode 100644 index 0000000..91aeacd --- /dev/null +++ b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/scored_a_interim_163.json @@ -0,0 +1,500 @@ +{ + "alerts_only": { + "count": 0, + "n": 149, + "rate": 0.0, + "wilson95": [ + 0, + 0.025134518351024853 + ] + }, + "breakdowns": { + "coverage": { + "engine_covered": { + "alerts_only": { + "count": 0, + "n": 96, + "rate": 0.0, + "wilson95": [ + 0, + 0.03847694748481595 + ] + }, + "catch": { + "count": 0, + "n": 4, + "rate": 0.0, + "wilson95": [ + 0, + 0.48990002040399916 + ] + }, + "false_alarm": { + "cluster95": [ + 0.0, + 0.033707865168539325 + ], + "count": 1, + "icc": 2.814649921584905e-17, + "n": 96, + "rate": 0.010416666666666666, + "wilson95": [ + 0.001841122818436635, + 0.056667554927112326 + ] + } + }, + "quick_only": { + "alerts_only": { + "count": 0, + "n": 53, + "rate": 0.0, + "wilson95": [ + 0, + 0.06758430445307662 + ] + }, + "catch": { + "count": 0, + "n": 8, + "rate": 0.0, + "wilson95": [ + 0, + 0.3244156195108769 + ] + }, + "false_alarm": { + "cluster95": [ + 0.0, + 0.0 + ], + "count": 0, + "icc": 0.0, + "n": 53, + "rate": 0.0, + "wilson95": [ + 0, + 0.06758430445307662 + ] + } + }, + "unanalysable": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "family": { + "access_control": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 3, + "rate": 0.0, + "wilson95": [ + 0, + 0.5615060804490177 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "output_encoding": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 4, + "rate": 0.0, + "wilson95": [ + 0, + 0.48990002040399916 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "path": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 3, + "rate": 0.0, + "wilson95": [ + 0, + 0.5615060804490177 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "unknown": { + "alerts_only": { + "count": 0, + "n": 149, + "rate": 0.0, + "wilson95": [ + 0, + 0.025134518351024853 + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + 0.0, + 0.02027027027027027 + ], + "count": 1, + "icc": -0.004445973772031825, + "n": 149, + "rate": 0.006711409395973154, + "wilson95": [ + 0.0011856820977550406, + 0.037034278959967463 + ] + } + }, + "untrusted_destination": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 2, + "rate": 0.0, + "wilson95": [ + 0, + 0.6576280471103807 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "method": { + "first_parent": { + "alerts_only": { + "count": 0, + "n": 149, + "rate": 0.0, + "wilson95": [ + 0, + 0.025134518351024853 + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + 0.0, + 0.02027027027027027 + ], + "count": 1, + "icc": -0.004445973772031825, + "n": 149, + "rate": 0.006711409395973154, + "wilson95": [ + 0.0011856820977550406, + 0.037034278959967463 + ] + } + }, + "fix_site": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "post_cutoff": { + "unknown": { + "alerts_only": { + "count": 0, + "n": 149, + "rate": 0.0, + "wilson95": [ + 0, + 0.025134518351024853 + ] + }, + "catch": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "false_alarm": { + "cluster95": [ + 0.0, + 0.02027027027027027 + ], + "count": 1, + "icc": -0.004445973772031825, + "n": 149, + "rate": 0.006711409395973154, + "wilson95": [ + 0.0011856820977550406, + 0.037034278959967463 + ] + } + } + }, + "stream": { + "advisory": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "alerts": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "quick": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + } + } + }, + "catch": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "confirmed": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "coverage": { + "engine_covered": 100, + "instrument_failure": 2, + "quick_only": 61, + "unanalysable": 0 + }, + "deadlines": [ + { + "applied_deadline_scale": 1.0, + "count": 163, + "deadline_scale": 1.0, + "deadline_seconds": 300, + "lane": "docker" + } + ], + "false_alarm": { + "cluster95": [ + 0.0, + 0.02027027027027027 + ], + "count": 1, + "icc": -0.004445973772031825, + "n": 149, + "rate": 0.006711409395973154, + "wilson95": [ + 0.0011856820977550406, + 0.037034278959967463 + ] + }, + "total": 163, + "unseen": { + "gate": "not_run" + } +} diff --git a/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/scored_b_filtered.json b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/scored_b_filtered.json new file mode 100644 index 0000000..4730587 --- /dev/null +++ b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/scored_b_filtered.json @@ -0,0 +1,500 @@ +{ + "alerts_only": { + "count": 130, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + }, + "breakdowns": { + "coverage": { + "engine_covered": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "quick_only": { + "alerts_only": { + "count": 130, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + }, + "catch": { + "count": 5, + "n": 12, + "rate": 0.4166666666666667, + "wilson95": [ + 0.19325746190524656, + 0.6804926643446272 + ] + }, + "false_alarm": { + "cluster95": [ + 0.4708333333333333, + 0.6166666666666667 + ], + "count": 130, + "icc": 0.022377467632178107, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + } + }, + "unanalysable": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "family": { + "access_control": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 3, + "rate": 0.0, + "wilson95": [ + 0, + 0.5615060804490177 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "output_encoding": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 2, + "n": 4, + "rate": 0.5, + "wilson95": [ + 0.15003570882017148, + 0.8499642911798285 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "path": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 2, + "n": 3, + "rate": 0.6666666666666666, + "wilson95": [ + 0.2076549551264879, + 0.9385096847238394 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "unknown": { + "alerts_only": { + "count": 130, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + 0.4708333333333333, + 0.6166666666666667 + ], + "count": 130, + "icc": 0.022377467632178107, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + } + }, + "untrusted_destination": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 1, + "n": 2, + "rate": 0.5, + "wilson95": [ + 0.09452865480086614, + 0.9054713451991339 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "method": { + "first_parent": { + "alerts_only": { + "count": 130, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + 0.4708333333333333, + 0.6166666666666667 + ], + "count": 130, + "icc": 0.022377467632178107, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + } + }, + "fix_site": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 5, + "n": 12, + "rate": 0.4166666666666667, + "wilson95": [ + 0.19325746190524656, + 0.6804926643446272 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "post_cutoff": { + "unknown": { + "alerts_only": { + "count": 130, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + }, + "catch": { + "count": 5, + "n": 12, + "rate": 0.4166666666666667, + "wilson95": [ + 0.19325746190524656, + 0.6804926643446272 + ] + }, + "false_alarm": { + "cluster95": [ + 0.4708333333333333, + 0.6166666666666667 + ], + "count": 130, + "icc": 0.022377467632178107, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + } + } + }, + "stream": { + "advisory": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "alerts": { + "count": 5, + "n": 12, + "rate": 0.4166666666666667, + "wilson95": [ + 0.19325746190524656, + 0.6804926643446272 + ] + }, + "quick": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + } + } + }, + "catch": { + "count": 5, + "n": 12, + "rate": 0.4166666666666667, + "wilson95": [ + 0.19325746190524656, + 0.6804926643446272 + ] + }, + "confirmed": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 5, + "n": 12, + "rate": 0.4166666666666667, + "wilson95": [ + 0.19325746190524656, + 0.6804926643446272 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "coverage": { + "engine_covered": 0, + "instrument_failure": 1, + "quick_only": 251, + "unanalysable": 0 + }, + "deadlines": [ + { + "applied_deadline_scale": null, + "count": 252, + "deadline_scale": null, + "deadline_seconds": null, + "lane": "unknown" + } + ], + "false_alarm": { + "cluster95": [ + 0.4708333333333333, + 0.6166666666666667 + ], + "count": 130, + "icc": 0.022377467632178107, + "n": 239, + "rate": 0.5439330543933054, + "wilson95": [ + 0.480589907402735, + 0.6058862153044123 + ] + }, + "total": 252, + "unseen": { + "gate": "not_run" + } +} diff --git a/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/scored_b_raw.json b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/scored_b_raw.json new file mode 100644 index 0000000..1863787 --- /dev/null +++ b/benchmarks/measurements/2026-10-12-valid-benchmark-pilot/scored_b_raw.json @@ -0,0 +1,500 @@ +{ + "alerts_only": { + "count": 221, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + }, + "breakdowns": { + "coverage": { + "engine_covered": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "quick_only": { + "alerts_only": { + "count": 221, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + }, + "catch": { + "count": 6, + "n": 12, + "rate": 0.5, + "wilson95": [ + 0.2537781703934222, + 0.7462218296065778 + ] + }, + "false_alarm": { + "cluster95": [ + 0.891213389121339, + 0.9583333333333334 + ], + "count": 221, + "icc": 0.006754645062417405, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + } + }, + "unanalysable": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "family": { + "access_control": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 0, + "n": 3, + "rate": 0.0, + "wilson95": [ + 0, + 0.5615060804490177 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "output_encoding": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 3, + "n": 4, + "rate": 0.75, + "wilson95": [ + 0.3006360524426366, + 0.9544139373553637 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "path": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 2, + "n": 3, + "rate": 0.6666666666666666, + "wilson95": [ + 0.2076549551264879, + 0.9385096847238394 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "unknown": { + "alerts_only": { + "count": 221, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + 0.891213389121339, + 0.9583333333333334 + ], + "count": 221, + "icc": 0.006754645062417405, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + } + }, + "untrusted_destination": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 1, + "n": 2, + "rate": 0.5, + "wilson95": [ + 0.09452865480086614, + 0.9054713451991339 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "method": { + "first_parent": { + "alerts_only": { + "count": 221, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + }, + "catch": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "false_alarm": { + "cluster95": [ + 0.891213389121339, + 0.9583333333333334 + ], + "count": 221, + "icc": 0.006754645062417405, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + } + }, + "fix_site": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 6, + "n": 12, + "rate": 0.5, + "wilson95": [ + 0.2537781703934222, + 0.7462218296065778 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + } + }, + "post_cutoff": { + "unknown": { + "alerts_only": { + "count": 221, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + }, + "catch": { + "count": 6, + "n": 12, + "rate": 0.5, + "wilson95": [ + 0.2537781703934222, + 0.7462218296065778 + ] + }, + "false_alarm": { + "cluster95": [ + 0.891213389121339, + 0.9583333333333334 + ], + "count": 221, + "icc": 0.006754645062417405, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + } + } + }, + "stream": { + "advisory": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "alerts": { + "count": 6, + "n": 12, + "rate": 0.5, + "wilson95": [ + 0.2537781703934222, + 0.7462218296065778 + ] + }, + "quick": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + } + } + }, + "catch": { + "count": 6, + "n": 12, + "rate": 0.5, + "wilson95": [ + 0.2537781703934222, + 0.7462218296065778 + ] + }, + "confirmed": { + "alerts_only": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + }, + "catch": { + "count": 6, + "n": 12, + "rate": 0.5, + "wilson95": [ + 0.2537781703934222, + 0.7462218296065778 + ] + }, + "false_alarm": { + "cluster95": [ + null, + null + ], + "count": 0, + "icc": null, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } + }, + "coverage": { + "engine_covered": 0, + "instrument_failure": 1, + "quick_only": 251, + "unanalysable": 0 + }, + "deadlines": [ + { + "applied_deadline_scale": null, + "count": 252, + "deadline_scale": null, + "deadline_seconds": null, + "lane": "unknown" + } + ], + "false_alarm": { + "cluster95": [ + 0.891213389121339, + 0.9583333333333334 + ], + "count": 221, + "icc": 0.006754645062417405, + "n": 239, + "rate": 0.9246861924686193, + "wilson95": [ + 0.8841031716603341, + 0.951832681175424 + ] + }, + "total": 252, + "unseen": { + "gate": "not_run" + } +}