From c5341d79a21bfae49ee157bdac19f967754f0be4 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:59:01 +0100 Subject: [PATCH] measure(prover): gate 1 on the 23-repo training pool: recall 65.2% (Wilson 44.9-81.2), 15.0 FA/100, 0 instrument failures; 4/5 on the hunt overlap incl. trigger.dev --- .kiro/specs/access-control-prover/tasks.md | 6 +- .kiro/steering/roadmap.md | 50 + .../2026-10-28-access-control-prover/NOTES.md | 114 ++ .../summary.json | 1186 +++++++++++++++++ 4 files changed, 1355 insertions(+), 1 deletion(-) create mode 100644 benchmarks/measurements/2026-10-28-access-control-prover/NOTES.md create mode 100644 benchmarks/measurements/2026-10-28-access-control-prover/summary.json diff --git a/.kiro/specs/access-control-prover/tasks.md b/.kiro/specs/access-control-prover/tasks.md index 0113ae0..7391f65 100644 --- a/.kiro/specs/access-control-prover/tasks.md +++ b/.kiro/specs/access-control-prover/tasks.md @@ -107,10 +107,14 @@ - _Depends: 4.1_ - [ ] 6. Measurement (maintainer, not Codex) -- [ ] 6.1 Run the arm on `benchmarks/valid/train/pool-train.toml` over the training clones; write +- [x] 6.1 Run the arm on `benchmarks/valid/train/pool-train.toml` over the training clones; write `benchmarks/measurements/-access-control-prover/` with NOTES, summary, coverage table; compare with the DSPy and hunter arms of the same pool - _Requirements: 5.2, 5.3, 7.2_ + - Done 2026-10-28: `benchmarks/measurements/2026-10-28-access-control-prover/` -- 23-repo + pool (483 changes, 93 locations), recall 65.2% (Wilson 44.9-81.2), 15.0 FA/100, 0 + instrument failures; on the 5 repos shared with the 2026-10-14 hunt arms, prover 4/5 vs + DSPy 2/5 and hunter 2/5 (union 4/5), including trigger.dev which neither arm found. ## Implementation Notes diff --git a/.kiro/steering/roadmap.md b/.kiro/steering/roadmap.md index 2b220ba..c4ae009 100644 --- a/.kiro/steering/roadmap.md +++ b/.kiro/steering/roadmap.md @@ -1,5 +1,55 @@ # Roadmap +## Roadmap -- measure the trace first (2026-10-28) + +This section supersedes the ordering below it. The older sections stay as history. It builds on the +2026-10-14 decision-strategy correction (steering `overview.md`): the trace decides, a judge reads it +zero-shot, the pool measures and never trains. + +### Decision (2026-10-28, maintainer session) + +The DSPy annotation loop was built and measured; its premise -- that shape annotations teach the hunter -- +was falsified by its own instruments (`benchmarks/measurements/2026-10-13-access-control-screen/`, +`2026-10-13-hunt-repeats/`, `2026-10-14-access-control-hunt/`). The surviving direction inverts the +augmentation: compute the deterministic security object (the ordered trace) first, and let the LLM judge +the residual question on it. Before anything generalizes, that direction gets its number. No new spec +starts until the gates below are read. + +### The sequence, each step a gate + +1. `access-control-prover` task 6.1: run the prover arm over `benchmarks/valid/train/pool-train.toml` -- + the first full-repository recall and false-alarm numbers for the trace. No model calls. Record the + provider error code for any `LMInvalidRequestError`-class failure (the 2026-10-14 run's failures sat on + 38 of 48 locations), and read the summary's `uncovered` repositories before calling any miss a logic + failure rather than a fact-coverage gap. + - **Read 2026-10-28** (`benchmarks/measurements/2026-10-28-access-control-prover/`): 23-repo pool, + 483 changes, 93 locations -- recall 65.2% (Wilson 44.9-81.2), 15.0 FA/100, 0 instrument failures. + On the 5 repos shared with the 2026-10-14 arms: prover 4/5 vs DSPy 2/5, hunter 2/5 (union 4/5), + including trigger.dev which neither arm found. Misses: 2 fact-coverage (seatunnel, byobu -- no + entry points), 6 covered-but-unflagged for gate 3. Gate 1 holds; proceed to gate 2. +2. Zero-shot judge pass over those traces ("is this entry meant to be public"): does it drop false alarms + without killing recall? The pool reports how often the judge is right (correction, rule 3). +3. Trace-component ablation (the amended `hunter-learning-loop` role -- selection, not fitting): which + trace components the judge needs to see, with intervals. +4. One pilot run under the at-most-once rule (`hunter-learning-loop` req 8.6): trace + judge over the 12 + verified positives and the ordinary pushes. Read the gate distance off the curve (recall at 10 false + alarms per 100 pushes, false alarms at 90% recall), per family -- the first such number for the new + direction. +5. Decision: + - access_control holds recall with false alarms moving toward the gate -> formalize the injection + security object (`engine-change-scoped-analysis` brief -> spec): the taint flow with argument + provenance as the deterministic object, the fact-proposal lane included. + - it fails -> fix the trace, do not generalize. A failure traced to fact coverage grows the fact + tables first; a logic failure revisits the prover. + +### What is deferred, explicitly + +- The injection-family spec waits for gate 5. Its fact-proposal lane absorbs the disciplined DSPy steer: + the LLM proposes trust boundaries and missing sinks, the engine verifies, and repetition promotes them + to closed facts -- never a prompt fitted on pool labels. +- `hunter-learning-loop` task 4.1 (training-set sourcing) stays parked: gates 1-4 need no fitted model. +- `pre-push-safety-net` wiring follows a search that clears the bar, unchanged from the 2026-10-13 order. + ## Roadmap to detection with the decision engine on kube-ax (2026-10-04) This section supersedes the ordering below it. The older sections stay as history. diff --git a/benchmarks/measurements/2026-10-28-access-control-prover/NOTES.md b/benchmarks/measurements/2026-10-28-access-control-prover/NOTES.md new file mode 100644 index 0000000..62c5712 --- /dev/null +++ b/benchmarks/measurements/2026-10-28-access-control-prover/NOTES.md @@ -0,0 +1,114 @@ +# Access-control prover arm -- training pool (gate 1) + +Date: 2026-10-28. Roadmap: `.kiro/steering/roadmap.md`, "measure the trace first", gate 1. + +## Setup + +- Arm: `benchmarks/valid/prover_arm.py`, deterministic trace + dominance check. No model + calls, no API key, no fitted components. +- Pool: `benchmarks/valid/train/pool-train.toml` (frozen 2026-10-14 09:31, freeze digest + e6a67487) -- 23 repositories x (1 vulnerability fix push + 20 ordinary pushes) = 483 + changes, 93 confirmed vulnerable locations. +- Clones: `~/ousast-results/valid/train/hist/`; repos mapping + `~/ousast-results/valid/train/repos_hist_pool23.json` -- rebuilt for this run. The + previous mapping (`repos_hist.json`, 29 entries from the 5-repo pool era) lacked 18 of + the 23 pool repositories and produced 378/441 `KeyError` instrument failures on a first + launch; those records were deleted and never summarized. `prover_arm._prove` records + only the exception type, so the missing key was invisible in records; it was recovered + from the run log. Follow-up candidate: record the exception message in instrument + failures. +- Command: `PYTHONPATH=$PWD .venv/bin/python -m benchmarks.valid.prover_arm run + benchmarks/valid/train/pool-train.toml --repos repos_hist_pool23.json --out + ~/ousast-results/prover/ac-train-1 --only all`, then `summarize` on the same out dir. +- Runtime: mean 27.9 s/change, median 0.9 s (ordinary pushes are fast; first touch of a + repo pays tree walk + framework scan, max 1063 s). 483/483 records, + **0 instrument failures**, no zero-byte read refusals. + +## Result (whole pool, 23 repos) + +Headline (from `summary.json`): on the training pool, recall 65.2% (Wilson 44.9-81.2), +false alarms 15.0 per 100 pushes, precision 17.9%; exploit-verified 0 of 78 (78 not yet +proved -- runtime proof is the async lane, out of scope here). + +| metric | value | +|---|---| +| recall (fix pushes) | 15/23 = 65.2% (Wilson 44.9-81.2; repo-clustered 95% 43.5-82.6) | +| location recall | 55/93 = 59.1% (Wilson 49.0-68.6) | +| false alarms | 69 on 460 ordinary pushes = 15.0 per 100 pushes | +| precision (suspicion level) | 15/84 = 17.9% | +| instrument failures | 0 | + +## Comparison with the DSPy and hunter arms (5 overlapping repos only) + +The 2026-10-14 hunt arms ran the old 5-repo pool (dependency-track, job-import, p4, +shopper, trigger.dev; digest 53f9f364, 48 locations, 100 ordinary pushes). The prover ran +the 23-repo pool; restrict to those 5 repositories for a like-for-like recall comparison. + +| arm | recall on 5 repos | notes | +|---|---|---| +| DSPy (2026-10-14) | 2/5 | 4/48 locations, 15.6 FA/100, 11 LMInvalidRequestError failures | +| hunter (2026-10-14) | 2/5 | different repos; union of the two arms 4/5, 99/100 ordinary pushes flagged | +| **prover (this run)** | **4/5** | misses p4 only; includes trigger.dev, which neither arm found | + +The prover alone matches the union of both learned arms on their own training pool, at +DSPy-level false-alarm rate (15.0 vs 15.6 FA/100) and with zero instrument failures and +zero model cost. False-alarm counts come from different ordinary-push sets (460 vs 90), +so treat the FA comparison as indicative, not paired. + +## Misses (8 fix pushes) + +Zero coverage (no entry points found -- framework facts absent, fix site never reached): +- apache/seatunnel (Java, JettyService) -- 0 entry points. +- friendsofflarum/byobu (Flarum JS) -- 0 entry points. + +Covered but not flagged: +- jenkinsci/analysis-core -- 172 entry points, 0 operations checked. +- jenkinsci/crx-content-package-deployer -- 12 checked, 2 discharged. +- jenkinsci/git -- 15 checked, 11 discharged. +- jenkinsci/global-build-stats -- 1 checked, 1 discharged. +- jenkinsci/p4 -- 38 checked, 3 discharged (the only overlap-repo miss). +- kimai/kimai -- 80 checked, 78 discharged; the fix's protected operation is dominated by + an existing check on every traced path, or the fix site is not entry-reachable under + current facts. + +Reading: 2 of 8 misses are fact-coverage failures (grow the entry/fact tables, gate 5's +"fact-coverage failure grows fact tables first"), 6 are trace/logic failures for gate 3 +ablation (what the checked-but-unflagged traces lack: entry reachability of the fix site, +missing operation kinds, or real dominance by an existing check). + +## Per-repository coverage + +| repository | recall | entry points | reached | ops checked | discharged | +|---|---|---|---|---|---| +| apache/seatunnel | 0/1 | 0 | 0 | 0 | 0 | +| friendsofflarum/byobu | 0/1 | 0 | 0 | 0 | 0 | +| jenkinsci/analysis-core | 0/1 | 172 | 7 | 0 | 0 | +| jenkinsci/azure-credentials | 1/1 | 100 | 29 | 6 | 0 | +| jenkinsci/azure-keyvault | 1/1 | 133 | 6 | 7 | 4 | +| jenkinsci/blueocean | 1/1 | 475 | 18 | 35 | 2 | +| jenkinsci/compuware-scm-downloader | 1/1 | 297 | 6 | 6 | 0 | +| jenkinsci/crx-content-package-deployer | 0/1 | 168 | 67 | 12 | 2 | +| jenkinsci/database | 1/1 | 93 | 38 | 8 | 5 | +| jenkinsci/dependency-track | 1/1 | 159 | 7 | 15 | 0 | +| jenkinsci/ec2 | 1/1 | 742 | 24 | 29 | 10 | +| jenkinsci/git | 0/1 | 584 | 24 | 15 | 11 | +| jenkinsci/global-build-stats | 0/1 | 448 | 32 | 1 | 1 | +| jenkinsci/hashicorp-vault | 1/1 | 179 | 15 | 17 | 2 | +| jenkinsci/jiratestresultreporter | 1/1 | 187 | 13 | 6 | 0 | +| jenkinsci/job-import | 1/1 | 103 | 43 | 17 | 9 | +| jenkinsci/opentelemetry | 1/1 | 157 | 9 | 5 | 2 | +| jenkinsci/p4 | 0/1 | 1018 | 35 | 38 | 3 | +| jenkinsci/rocketchatnotifier | 1/1 | 57 | 10 | 6 | 0 | +| kimai/kimai | 0/1 | 1951 | 78 | 80 | 78 | +| mage-ai/mage-ai | 1/1 | 214 | 1 | 63 | 0 | +| shopperlabs/shopper | 1/1 | 2645 | 53 | 115 | 1 | +| triggerdotdev/trigger.dev | 1/1 | 7164 | 20 | 14 | 0 | + +Raw records: `~/ousast-results/prover/ac-train-1/` (483 records + `summary.json`, also +copied here). Watcher log: `~/ousast-results/prover/ac-train-1.log.watch`. + +## Gate decision + +Gate 1 passes its purpose: the trace-based prover holds recall the learned arms only +reached as a union, at comparable false alarms, with a clean instrument. Next gate (2): +zero-shot judge pass over these same traces where judgment remains. diff --git a/benchmarks/measurements/2026-10-28-access-control-prover/summary.json b/benchmarks/measurements/2026-10-28-access-control-prover/summary.json new file mode 100644 index 0000000..b78186f --- /dev/null +++ b/benchmarks/measurements/2026-10-28-access-control-prover/summary.json @@ -0,0 +1,1186 @@ +{ + "changes_over_120_seconds": [ + { + "id": "ea5e6a56b2b433f9751c27533cc830fae1a66ae4df96fc8d6af7caee1c8235d3", + "seconds": 160.82508028903976 + }, + { + "id": "a660b8c23d6b61ab768a84c00582a0a231e1f0089c44980de934752f6a6022b8", + "seconds": 1062.5694906380959 + }, + { + "id": "bada9b8aaceaff26178d1940f7f6032b24d213071c54c30d2e7ce929e557df90", + "seconds": 139.3456460563466 + }, + { + "id": "514850869e2c19bfc55dbc75b2c1e1766249e12f04385b1de7b4d116b6b209fa", + "seconds": 356.29877026518807 + }, + { + "id": "1d77199cf964d7f168f3f0c157646f52aef7e3931b8da2f9fa829e1e32931964", + "seconds": 519.9195040278137 + }, + { + "id": "720017c40d9dd8ad541f00c2577b2d1676ba3dfcaa997166a1065dbb36b42231", + "seconds": 325.4053673329763 + }, + { + "id": "75f92ee98982d750540239481ee8aa4258ec3cc3cb64cce6ca19835d9fcf2d99", + "seconds": 376.9615895450115 + }, + { + "id": "f664ff98347b54ace35a50a499c77cfe3c6e06158e78f76f20b9d9317a29bb94", + "seconds": 438.4376673758961 + }, + { + "id": "e934b0222e338b768bfc594668434850f18e3a7ee35a76fb35c19898d4924349", + "seconds": 485.50487636821344 + }, + { + "id": "c0fca5070cee86d26832638230b364f45d245c6e1a77333d38ee393832291e13", + "seconds": 860.2654483858496 + }, + { + "id": "c2c52c1bb6b1f0d70b29b7b64009d6ca85c6e4d3659a195a904d577f6b02c886", + "seconds": 631.7540220390074 + }, + { + "id": "da89557ec3cada71aed38482d8269959d56e9a37877f66d69cf5b3905d0cbcff", + "seconds": 177.52998107904568 + }, + { + "id": "c1859d8c2376e25dd46024b336bb8106a702f8139437934cf48d282cf0f396cb", + "seconds": 132.389600626193 + }, + { + "id": "ea9ed8aad6259c9ea8ee1f8227a981710e8b4b1da838e74d6e68075455e78631", + "seconds": 696.5516875013709 + }, + { + "id": "58a2ffe30e0bc6a166a01fe016ebf43b0c0f3385b0f4a2e8578d3e039371c17e", + "seconds": 397.4514326550998 + }, + { + "id": "59327d70f193338a3ae990177f6fa8aece8e6913e6e5d3a27888d383edb56165", + "seconds": 243.8948559788987 + }, + { + "id": "5c9a822532e45241cb40e12ba6437837bf0301a6b2bead1261fbaa90e028f211", + "seconds": 291.92363393399864 + }, + { + "id": "a505c341e4c5e363b3050d1523bd8f6868c10e3cd5ed383d3c8e155b49bf9a70", + "seconds": 385.80087945610285 + }, + { + "id": "5c73c7dac092be000911ad6d4327156dbdc7a02d3d22545cfe47a25b67ce76ec", + "seconds": 612.4415660048835 + }, + { + "id": "b74b7098107cb5533f7fb7cea6a8b9532632b3b0038c15596260175f6c91d6dc", + "seconds": 293.1561822928488 + } + ], + "covered_recall": { + "count": 15, + "n": 21, + "rate": 0.7142857142857143, + "wilson95": [ + 0.500432261771624, + 0.8618632425437179 + ] + }, + "exploit_verified": { + "count": 0, + "flagged_positive_pushes": 15, + "flagged_positives": 78, + "n": 0, + "not_yet_proved": [ + "jenkinsci-azure-credentials-plugin-0dab78b4ed2c29bb", + "jenkinsci-azure-credentials-plugin-144d42edd4fea476", + "jenkinsci-azure-credentials-plugin-81c51344b9d31af8", + "jenkinsci-azure-credentials-plugin-a83f939fb055e6a0", + "jenkinsci-azure-credentials-plugin-af7645f7ae48e24c", + "jenkinsci-azure-credentials-plugin-bd1228731e6d22aa", + "jenkinsci-azure-credentials-plugin-d3c1129ce0199756", + "jenkinsci-azure-keyvault-plugin-a9975f6d8b4f1568", + "jenkinsci-azure-keyvault-plugin-abd182ce20617255", + "jenkinsci-blueocean-plugin-5aa4f2eb0a53ebfe", + "jenkinsci-blueocean-plugin-e4ad2dadd845ec00", + "jenkinsci-compuware-scm-downloader-plugin-29636a1534a39217", + "jenkinsci-compuware-scm-downloader-plugin-316557e9fdf67223", + "jenkinsci-compuware-scm-downloader-plugin-4b64810cb01a07f5", + "jenkinsci-compuware-scm-downloader-plugin-936a096a4a941ee5", + "jenkinsci-compuware-scm-downloader-plugin-c416a32628e63296", + "jenkinsci-database-plugin-6a8e141d0c9eaab2", + "jenkinsci-database-plugin-d2e038dce743f4f4", + "jenkinsci-database-plugin-fdb5a7fd01260ec1", + "jenkinsci-dependency-track-plugin-673cc48b4f75ebf5", + "jenkinsci-dependency-track-plugin-90b566be2c3aebff", + "jenkinsci-dependency-track-plugin-9e8488d96f7c0424", + "jenkinsci-ec2-plugin-6058527110940076", + "jenkinsci-ec2-plugin-92278d8bf0b7f60a", + "jenkinsci-hashicorp-vault-plugin-4b8dac7099e598b2", + "jenkinsci-hashicorp-vault-plugin-69b136827df9a5c0", + "jenkinsci-hashicorp-vault-plugin-7782cfb6ae3f0339", + "jenkinsci-hashicorp-vault-plugin-9f7965aeec3cc869", + "jenkinsci-hashicorp-vault-plugin-c76a541ee22b2ef9", + "jenkinsci-hashicorp-vault-plugin-e7d568b004427b1d", + "jenkinsci-jiratestresultreporter-plugin-28213733c6b799eb", + "jenkinsci-job-import-plugin-07e0e7023a46cf4d", + "jenkinsci-job-import-plugin-5509a356384e862e", + "jenkinsci-opentelemetry-plugin-62ce6dc3488e0d80", + "jenkinsci-opentelemetry-plugin-c258c6ed37f2c061", + "jenkinsci-opentelemetry-plugin-e7dc96d707803ae7", + "jenkinsci-opentelemetry-plugin-f0ff00bf7375eb29", + "jenkinsci-rocketchatnotifier-plugin-16d28ce862d5a528", + "mage-ai-mage-ai-c9810ee637c308c0", + "mage-ai-mage-ai-d1ee6d54adc38f57", + "shopperlabs-shopper-02254beed509364c", + "shopperlabs-shopper-047e62abed0b91fa", + "shopperlabs-shopper-052f88a347caf2eb", + "shopperlabs-shopper-064f0b5f8937aae5", + "shopperlabs-shopper-0fd6e8d87dd0ed82", + "shopperlabs-shopper-1112cc61420816ff", + "shopperlabs-shopper-13fdb238d0d8e573", + "shopperlabs-shopper-22487ec06a8cf50a", + "shopperlabs-shopper-2405022067cc6870", + "shopperlabs-shopper-29811dac402a0d13", + "shopperlabs-shopper-2aeaefee52274ee8", + "shopperlabs-shopper-5481a94c34f42fbf", + "shopperlabs-shopper-5a3dcf2a6b6b16c8", + "shopperlabs-shopper-5e6161ace0cb4b97", + "shopperlabs-shopper-61e951b6bcd03186", + "shopperlabs-shopper-7cc69fb293ec0e10", + "shopperlabs-shopper-8769d9915b8295a9", + "shopperlabs-shopper-88497bd11908cf4a", + "shopperlabs-shopper-8b7021ee40c51b56", + "shopperlabs-shopper-91023b586e4ab747", + "shopperlabs-shopper-a413505de3d037f8", + "shopperlabs-shopper-a82b452965e2ffe5", + "shopperlabs-shopper-ab37699016e7189c", + "shopperlabs-shopper-b37cd0c75b594760", + "shopperlabs-shopper-cd45e0d6604bb846", + "shopperlabs-shopper-d05ad8c2776691be", + "shopperlabs-shopper-d26bdca35566e799", + "shopperlabs-shopper-deb90b541bcac1aa", + "shopperlabs-shopper-df2391064e3c3332", + "shopperlabs-shopper-f3cb66743db8d5f1", + "shopperlabs-shopper-fddd0d6c3c4501aa", + "triggerdotdev-trigger.dev-0eb72665259b13c2", + "triggerdotdev-trigger.dev-124d417d0ae18fbe", + "triggerdotdev-trigger.dev-2240070f2c86924b", + "triggerdotdev-trigger.dev-30dec4b7031de737", + "triggerdotdev-trigger.dev-5b0c0ef74fdf5bc3", + "triggerdotdev-trigger.dev-5f1cf28e3ac84980", + "triggerdotdev-trigger.dev-eebfca9646e01b1c" + ], + "rate": null, + "status": "not yet proved", + "wilson95": [ + null, + null + ] + }, + "false_alarms_per_100_pushes": { + "cluster95": [ + 8.695652173913043, + 21.521739130434785 + ], + "count": 69, + "icc": 0.144640754781103, + "n": 460, + "rate": 15.0, + "wilson95": [ + 12.027398310244092, + 18.552351328425665 + ] + }, + "headline": "On the training pool: recall 65.2% (Wilson 44.9-81.2), false alarms 15.0 per 100 pushes, precision 17.9%; exploit-verified 0 of 78 (78 not yet proved).", + "instrument_failures": 0, + "location_recall": { + "count": 55, + "n": 93, + "rate": 0.5913978494623656, + "wilson95": [ + 0.4897977403042717, + 0.6857466538403933 + ] + }, + "per_framework": { + "jaxrs": { + "changes": 68, + "entry_points": 1408, + "files": 373, + "operations_checked": 70, + "operations_discharged": 20, + "reached": 56, + "unchecked_by_reason": { + "FileNotFoundError": 5, + "ValueError": 790, + "_Absent": 39, + "framework_unrecognised": 297, + "function_unavailable": 62, + "language_unsupported": 93, + "long_line": 1, + "parse_failed": 80, + "unreached": 34 + }, + "unreached": 293 + }, + "laravel": { + "changes": 17, + "entry_points": 1864, + "files": 190, + "operations_checked": 65, + "operations_discharged": 0, + "reached": 31, + "unchecked_by_reason": { + "FileNotFoundError": 12, + "ValueError": 3, + "_Absent": 16, + "framework_unrecognised": 91, + "function_unavailable": 68, + "too_large": 2 + }, + "unreached": 88 + }, + "livewire": { + "changes": 20, + "entry_points": 2641, + "files": 638, + "operations_checked": 115, + "operations_discharged": 1, + "reached": 53, + "unchecked_by_reason": { + "FileNotFoundError": 14, + "ValueError": 3, + "_Absent": 14, + "excluded_dir": 3, + "framework_unrecognised": 113, + "function_unavailable": 66, + "language_unsupported": 3, + "too_large": 2, + "unreached": 6 + }, + "unreached": 115 + }, + "python": { + "changes": 21, + "entry_points": 214, + "files": 22, + "operations_checked": 63, + "operations_discharged": 0, + "reached": 1, + "unchecked_by_reason": { + "FileNotFoundError": 12, + "ValueError": 2454, + "_Absent": 4064, + "framework_unrecognised": 214, + "function_unavailable": 10, + "language_unsupported": 4, + "long_line": 116, + "parse_failed": 97, + "too_large": 2 + }, + "unreached": 96 + }, + "remix": { + "changes": 21, + "entry_points": 7164, + "files": 6153, + "operations_checked": 14, + "operations_discharged": 0, + "reached": 20, + "unchecked_by_reason": { + "ValueError": 335, + "_Absent": 7, + "framework_unrecognised": 171, + "function_unavailable": 21, + "language_unsupported": 18, + "parse_failed": 7, + "too_large": 1, + "unreached": 63 + }, + "unreached": 233 + }, + "stapler": { + "changes": 345, + "entry_points": 5072, + "files": 2009, + "operations_checked": 223, + "operations_discharged": 51, + "reached": 383, + "unchecked_by_reason": { + "FileNotFoundError": 23, + "ValueError": 790, + "_Absent": 39, + "excluded_dir": 1, + "framework_unrecognised": 1285, + "function_unavailable": 175, + "language_unsupported": 354, + "long_line": 2, + "parse_failed": 80, + "unreached": 442 + }, + "unreached": 1642 + }, + "symfony": { + "changes": 21, + "entry_points": 1951, + "files": 470, + "operations_checked": 80, + "operations_discharged": 78, + "reached": 78, + "unchecked_by_reason": { + "FileNotFoundError": 1, + "ValueError": 1, + "_Absent": 16, + "excluded_dir": 30, + "framework_unrecognised": 227, + "function_unavailable": 36, + "language_unsupported": 61, + "unreached": 1 + }, + "unreached": 198 + }, + "tornado": { + "changes": 21, + "entry_points": 214, + "files": 76, + "operations_checked": 63, + "operations_discharged": 0, + "reached": 1, + "unchecked_by_reason": { + "FileNotFoundError": 12, + "ValueError": 2454, + "_Absent": 4064, + "framework_unrecognised": 214, + "function_unavailable": 10, + "language_unsupported": 4, + "long_line": 116, + "parse_failed": 97, + "too_large": 2 + }, + "unreached": 96 + }, + "typescript": { + "changes": 5, + "entry_points": 1620, + "files": 26, + "operations_checked": 7, + "operations_discharged": 0, + "reached": 10, + "unchecked_by_reason": { + "ValueError": 69, + "framework_unrecognised": 45, + "function_unavailable": 14, + "language_unsupported": 7, + "parse_failed": 1, + "unreached": 2 + }, + "unreached": 46 + } + }, + "per_repository": { + "apache/seatunnel": { + "changes": 21, + "entry_points": 0, + "files_by_language": { + "java": 74116, + "javascript": 42, + "python": 375, + "typescript": 1153 + }, + "frameworks_recognised": {}, + "operations_checked": 0, + "operations_discharged": 0, + "reached": 0, + "recall": { + "count": 0, + "n": 1, + "rate": 0.0, + "wilson95": [ + 0, + 0.7934567085261071 + ] + }, + "status": "uncovered", + "unchecked_by_reason": { + "FileNotFoundError": 2, + "framework_unrecognised": 397, + "function_unavailable": 7, + "language_unsupported": 70 + }, + "unreached": 272 + }, + "friendsofflarum/byobu": { + "changes": 21, + "entry_points": 0, + "files_by_language": { + "javascript": 576, + "php": 825, + "typescript": 22 + }, + "frameworks_recognised": {}, + "operations_checked": 0, + "operations_discharged": 0, + "reached": 0, + "recall": { + "count": 0, + "n": 1, + "rate": 0.0, + "wilson95": [ + 0, + 0.7934567085261071 + ] + }, + "status": "uncovered", + "unchecked_by_reason": { + "FileNotFoundError": 5, + "ValueError": 25, + "_Absent": 7, + "excluded_dir": 10, + "framework_unrecognised": 73, + "function_unavailable": 18, + "language_unsupported": 15, + "parse_failed": 1 + }, + "unreached": 72 + }, + "jenkinsci/analysis-core-plugin": { + "changes": 21, + "entry_points": 172, + "files_by_language": { + "java": 2100, + "javascript": 58 + }, + "frameworks_recognised": { + "stapler": 86 + }, + "operations_checked": 0, + "operations_discharged": 0, + "reached": 7, + "recall": { + "count": 0, + "n": 1, + "rate": 0.0, + "wilson95": [ + 0, + 0.7934567085261071 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "framework_unrecognised": 54, + "function_unavailable": 5, + "language_unsupported": 6, + "unreached": 10 + }, + "unreached": 64 + }, + "jenkinsci/azure-credentials-plugin": { + "changes": 21, + "entry_points": 100, + "files_by_language": { + "java": 242 + }, + "frameworks_recognised": { + "jaxrs": 16, + "stapler": 71 + }, + "operations_checked": 6, + "operations_discharged": 0, + "reached": 29, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 1, + "framework_unrecognised": 63, + "function_unavailable": 10, + "language_unsupported": 30, + "unreached": 18 + }, + "unreached": 78 + }, + "jenkinsci/azure-keyvault-plugin": { + "changes": 21, + "entry_points": 133, + "files_by_language": { + "java": 283 + }, + "frameworks_recognised": { + "jaxrs": 31, + "stapler": 69 + }, + "operations_checked": 7, + "operations_discharged": 4, + "reached": 6, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 6, + "framework_unrecognised": 74, + "function_unavailable": 1, + "language_unsupported": 26, + "unreached": 57 + }, + "unreached": 113 + }, + "jenkinsci/blueocean-plugin": { + "changes": 21, + "entry_points": 475, + "files_by_language": { + "java": 7489, + "javascript": 20481, + "typescript": 134 + }, + "frameworks_recognised": { + "jaxrs": 245, + "stapler": 110 + }, + "operations_checked": 35, + "operations_discharged": 2, + "reached": 18, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 4, + "ValueError": 790, + "_Absent": 39, + "framework_unrecognised": 113, + "function_unavailable": 36, + "language_unsupported": 53, + "parse_failed": 80, + "unreached": 1 + }, + "unreached": 113 + }, + "jenkinsci/compuware-scm-downloader-plugin": { + "changes": 21, + "entry_points": 297, + "files_by_language": { + "java": 221 + }, + "frameworks_recognised": { + "stapler": 58 + }, + "operations_checked": 6, + "operations_discharged": 0, + "reached": 6, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "framework_unrecognised": 60, + "function_unavailable": 15, + "language_unsupported": 24, + "unreached": 27 + }, + "unreached": 81 + }, + "jenkinsci/crx-content-package-deployer-plugin": { + "changes": 21, + "entry_points": 168, + "files_by_language": { + "java": 382 + }, + "frameworks_recognised": { + "stapler": 93 + }, + "operations_checked": 12, + "operations_discharged": 2, + "reached": 67, + "recall": { + "count": 0, + "n": 1, + "rate": 0.0, + "wilson95": [ + 0, + 0.7934567085261071 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 3, + "framework_unrecognised": 54, + "function_unavailable": 29, + "language_unsupported": 68, + "unreached": 152 + }, + "unreached": 199 + }, + "jenkinsci/database-plugin": { + "changes": 21, + "entry_points": 93, + "files_by_language": { + "java": 319 + }, + "frameworks_recognised": { + "jaxrs": 14, + "stapler": 51 + }, + "operations_checked": 8, + "operations_discharged": 5, + "reached": 38, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 4, + "framework_unrecognised": 71, + "function_unavailable": 10, + "language_unsupported": 24, + "long_line": 1, + "unreached": 9 + }, + "unreached": 79 + }, + "jenkinsci/dependency-track-plugin": { + "changes": 21, + "entry_points": 159, + "files_by_language": { + "java": 470, + "javascript": 63 + }, + "frameworks_recognised": { + "jaxrs": 17, + "stapler": 27 + }, + "operations_checked": 15, + "operations_discharged": 0, + "reached": 7, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "framework_unrecognised": 69, + "function_unavailable": 8, + "language_unsupported": 26, + "unreached": 5 + }, + "unreached": 58 + }, + "jenkinsci/ec2-plugin": { + "changes": 21, + "entry_points": 742, + "files_by_language": { + "java": 1106, + "javascript": 6, + "python": 16 + }, + "frameworks_recognised": { + "jaxrs": 36, + "stapler": 127 + }, + "operations_checked": 29, + "operations_discharged": 10, + "reached": 24, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "framework_unrecognised": 65, + "function_unavailable": 7, + "language_unsupported": 2, + "unreached": 18 + }, + "unreached": 69 + }, + "jenkinsci/git-plugin": { + "changes": 21, + "entry_points": 584, + "files_by_language": { + "java": 2881, + "javascript": 33 + }, + "frameworks_recognised": { + "stapler": 375 + }, + "operations_checked": 15, + "operations_discharged": 11, + "reached": 24, + "recall": { + "count": 0, + "n": 1, + "rate": 0.0, + "wilson95": [ + 0, + 0.7934567085261071 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 2, + "framework_unrecognised": 142, + "function_unavailable": 8, + "language_unsupported": 3, + "long_line": 1, + "unreached": 16 + }, + "unreached": 153 + }, + "jenkinsci/global-build-stats-plugin": { + "changes": 21, + "entry_points": 448, + "files_by_language": { + "java": 925, + "javascript": 169 + }, + "frameworks_recognised": { + "stapler": 21 + }, + "operations_checked": 1, + "operations_discharged": 1, + "reached": 32, + "recall": { + "count": 0, + "n": 1, + "rate": 0.0, + "wilson95": [ + 0, + 0.7934567085261071 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 1, + "framework_unrecognised": 72, + "function_unavailable": 6, + "language_unsupported": 9, + "unreached": 7 + }, + "unreached": 79 + }, + "jenkinsci/hashicorp-vault-plugin": { + "changes": 21, + "entry_points": 179, + "files_by_language": { + "java": 733 + }, + "frameworks_recognised": { + "stapler": 115 + }, + "operations_checked": 17, + "operations_discharged": 2, + "reached": 15, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 1, + "framework_unrecognised": 135, + "function_unavailable": 3, + "language_unsupported": 11, + "unreached": 3 + }, + "unreached": 134 + }, + "jenkinsci/jiratestresultreporter-plugin": { + "changes": 21, + "entry_points": 187, + "files_by_language": { + "java": 408, + "javascript": 11 + }, + "frameworks_recognised": { + "stapler": 108 + }, + "operations_checked": 6, + "operations_discharged": 0, + "reached": 13, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "framework_unrecognised": 66, + "function_unavailable": 16, + "language_unsupported": 8, + "unreached": 24 + }, + "unreached": 89 + }, + "jenkinsci/job-import-plugin": { + "changes": 21, + "entry_points": 103, + "files_by_language": { + "java": 179 + }, + "frameworks_recognised": { + "jaxrs": 6, + "stapler": 27 + }, + "operations_checked": 17, + "operations_discharged": 9, + "reached": 43, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 2, + "framework_unrecognised": 60, + "function_unavailable": 8, + "language_unsupported": 30, + "unreached": 17 + }, + "unreached": 75 + }, + "jenkinsci/opentelemetry-plugin": { + "changes": 21, + "entry_points": 157, + "files_by_language": { + "java": 1547 + }, + "frameworks_recognised": { + "stapler": 91 + }, + "operations_checked": 5, + "operations_discharged": 2, + "reached": 9, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 2, + "framework_unrecognised": 77, + "function_unavailable": 10, + "language_unsupported": 12, + "unreached": 18 + }, + "unreached": 95 + }, + "jenkinsci/p4-plugin": { + "changes": 21, + "entry_points": 1018, + "files_by_language": { + "java": 2292, + "javascript": 35 + }, + "frameworks_recognised": { + "jaxrs": 8, + "stapler": 551 + }, + "operations_checked": 38, + "operations_discharged": 3, + "reached": 35, + "recall": { + "count": 0, + "n": 1, + "rate": 0.0, + "wilson95": [ + 0, + 0.7934567085261071 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "excluded_dir": 1, + "framework_unrecognised": 96, + "function_unavailable": 2, + "language_unsupported": 10, + "unreached": 20 + }, + "unreached": 112 + }, + "jenkinsci/rocketchatnotifier-plugin": { + "changes": 21, + "entry_points": 57, + "files_by_language": { + "java": 437 + }, + "frameworks_recognised": { + "stapler": 29 + }, + "operations_checked": 6, + "operations_discharged": 0, + "reached": 10, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 1, + "framework_unrecognised": 56, + "function_unavailable": 5, + "language_unsupported": 19, + "unreached": 40 + }, + "unreached": 93 + }, + "kimai/kimai": { + "changes": 21, + "entry_points": 1951, + "files_by_language": { + "javascript": 708, + "php": 12785 + }, + "frameworks_recognised": { + "symfony": 470 + }, + "operations_checked": 80, + "operations_discharged": 78, + "reached": 78, + "recall": { + "count": 0, + "n": 1, + "rate": 0.0, + "wilson95": [ + 0, + 0.7934567085261071 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 1, + "ValueError": 1, + "_Absent": 16, + "excluded_dir": 30, + "framework_unrecognised": 227, + "function_unavailable": 36, + "language_unsupported": 61, + "unreached": 1 + }, + "unreached": 198 + }, + "mage-ai/mage-ai": { + "changes": 21, + "entry_points": 214, + "files_by_language": { + "javascript": 13713, + "python": 26766, + "typescript": 17808 + }, + "frameworks_recognised": { + "python": 22, + "tornado": 76 + }, + "operations_checked": 63, + "operations_discharged": 0, + "reached": 1, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 12, + "ValueError": 2454, + "_Absent": 4064, + "framework_unrecognised": 214, + "function_unavailable": 10, + "language_unsupported": 4, + "long_line": 116, + "parse_failed": 97, + "too_large": 2 + }, + "unreached": 96 + }, + "shopperlabs/shopper": { + "changes": 21, + "entry_points": 2645, + "files_by_language": { + "javascript": 177, + "php": 9485, + "typescript": 310 + }, + "frameworks_recognised": { + "laravel": 190, + "livewire": 638 + }, + "operations_checked": 115, + "operations_discharged": 1, + "reached": 53, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "FileNotFoundError": 14, + "ValueError": 3, + "_Absent": 16, + "excluded_dir": 3, + "framework_unrecognised": 114, + "function_unavailable": 71, + "language_unsupported": 3, + "too_large": 2, + "unreached": 6 + }, + "unreached": 116 + }, + "triggerdotdev/trigger.dev": { + "changes": 21, + "entry_points": 7164, + "files_by_language": { + "javascript": 500, + "python": 15, + "typescript": 49514 + }, + "frameworks_recognised": { + "remix": 6153, + "typescript": 26 + }, + "operations_checked": 14, + "operations_discharged": 0, + "reached": 20, + "recall": { + "count": 1, + "n": 1, + "rate": 1.0, + "wilson95": [ + 0.20654329147389294, + 1 + ] + }, + "status": "covered", + "unchecked_by_reason": { + "ValueError": 335, + "_Absent": 7, + "framework_unrecognised": 171, + "function_unavailable": 21, + "language_unsupported": 18, + "parse_failed": 7, + "too_large": 1, + "unreached": 63 + }, + "unreached": 233 + } + }, + "precision": { + "count": 15, + "n": 84, + "rate": 0.17857142857142858, + "wilson95": [ + 0.1113109090339025, + 0.2739461901081896 + ] + }, + "recall": { + "cluster95": [ + 0.43478260869565216, + 0.8260869565217391 + ], + "count": 15, + "n": 23, + "rate": 0.6521739130434783, + "wilson95": [ + 0.4488997690689164, + 0.8118894536301776 + ] + }, + "seconds_per_change": { + "maximum": 1062.5694906380959, + "mean": 27.87878479132637, + "median": 0.878022441174835, + "n": 483 + }, + "uncovered_recall": { + "count": 0, + "n": 2, + "rate": 0.0, + "wilson95": [ + 0, + 0.6576280471103807 + ] + }, + "uncovered_repositories": [ + "apache/seatunnel", + "friendsofflarum/byobu" + ] +}