From d063167b28ed7db2335737fdb0dae0555225ea85 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:37:19 +0100 Subject: [PATCH] feat(access-control-prover): 13.3 four-arm development regression --- .kiro/specs/access-control-prover/tasks.md | 2 +- .../NOTES.md | 125 +++++ .../summary.json | 489 ++++++++++++++++++ benchmarks/valid/arm_regression.py | 335 ++++++++++++ tests/test_admission_scoring.py | 4 + tests/test_arm_regression.py | 260 ++++++++++ 6 files changed, 1214 insertions(+), 1 deletion(-) create mode 100644 benchmarks/measurements/2026-10-30-access-control-four-arm/NOTES.md create mode 100644 benchmarks/measurements/2026-10-30-access-control-four-arm/summary.json create mode 100644 benchmarks/valid/arm_regression.py create mode 100644 tests/test_arm_regression.py diff --git a/.kiro/specs/access-control-prover/tasks.md b/.kiro/specs/access-control-prover/tasks.md index 64af83f..5c0280e 100644 --- a/.kiro/specs/access-control-prover/tasks.md +++ b/.kiro/specs/access-control-prover/tasks.md @@ -314,7 +314,7 @@ its final resolution, dominance, fact-acquisition and measurement semantics with - _Requirements: 1.1, 1.2, 1.3, 2.1, 2.2, 2.5, 2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 4.1, 4.2, 4.3, 10.6_ - _Boundary: Prover Integration Validation_ - _Depends: 10.6, 11.7_ -- [ ] 13.3 Run the four-arm development regression +- [x] 13.3 Run the four-arm development regression - Run all four registered arms on the immutable current development population after proving nonzero input reads. - Publish analyzer/fact attribution, stage funnel, three-way judgments, budget result and comparison to the 2026-10-28 baseline; label the result development-only. diff --git a/benchmarks/measurements/2026-10-30-access-control-four-arm/NOTES.md b/benchmarks/measurements/2026-10-30-access-control-four-arm/NOTES.md new file mode 100644 index 0000000..145cf55 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-four-arm/NOTES.md @@ -0,0 +1,125 @@ +# Access-control four-arm development regression (task 13.3) + +Date: 2026-10-30. Spec: `.kiro/specs/access-control-prover/` task 13.3 (Reqs 5.1-5.7, 6.1, 7.3, 11.8, 12.9). +Result role: **development regression evidence only** -- population used to choose facts and analyzer +behavior; not an admission claim (Req 5.7, 12.9). + +## Setup + +- Arms: four registered per Req 5.5 (old/new analyzer crossed with old/new fact set). + - Baseline bundle: analyzer revision `c5341d79a21bfae49ee157bdac19f967754f0be4` (the 2026-10-28 gate-1 + commit), fact artifact digest `3eb5bf699434`, materialised under + `~/ousast-results/prover/ac-four-arm-1/bundles/baseline/`. + - Candidate bundle: analyzer revision `9c95f842f7263710ec7d106794b37eb94a45425d` (HEAD at run time), + same fact artifact digest `3eb5bf699434`. +- Fact identity, proven before any arm ran: the 18 ruleset TOMLs under `src/openultrasast/ruleset` are + byte-identical between the baseline revision and HEAD (manifest digest `e7db4031d3d5`, disk == HEAD == + baseline ref). The amendment cycle is therefore analyzer-only, and the cross arms are input-identical to + their parents by digest proof: + - `baseline-analyzer/candidate-facts` == `baseline-analyzer/baseline-facts` (identical inputs), + - `candidate-analyzer/baseline-facts` == `candidate-analyzer/candidate-facts` (identical inputs). + Determinism is spot-verified by re-proving sampled changes and diffing judgments (below). +- Population: `benchmarks/valid/train/pool-train.toml`, freeze-verified through `load_slice` (the same + frozen 23-repo pool as the 2026-10-28 run; 483 changes = 23 fix pushes + 460 ordinary pushes). +- Clones: `~/ousast-results/valid/train/hist/`, repos mapping + `~/ousast-results/valid/train/repos_hist_pool23.json`. +- Nonzero input reads: the runner records a byte table per change (`instrument.file_bytes`, + `changed_bytes`, `hook_bytes_read`), refuses zero-byte reads of tracked files, and allows a file git + holds empty only as a counted `empty_files` entry -- the same instrument contract as the 2026-10-28 run. + Every adapted baseline record kept `hook_bytes_read > 0`; the summary refuses on any zero-byte entry. + +## Execution + +- `baseline-analyzer/baseline-facts`: the immutable 2026-10-28 records themselves + (`~/ousast-results/prover/ac-train-1/`, 483 records, 0 instrument failures at run time), adapted by + `benchmarks.valid.arm_regression.adapt_baseline_record` with an explicit `vocabulary: binary` marker: + unguarded := findings count, discharged := recorded discharge count, indeterminate := 0 (the baseline + analyzer could not express it). Research decision: the 2026-10-28 numbers remain the immutable baseline; + a rerun on the same repositories is regression evidence. +- `candidate-analyzer/candidate-facts`: executed 2026-10-30 from the working tree (clean, HEAD), strict + judgment lane (`judgment_mode=True`, three-way vocabulary), two resumable shards over the sorted change + ids (round-robin), records under `~/ousast-results/prover/ac-four-arm-1/cacfon/`. +- Cross arms: registered aliases by proven input identity (digest proof above) plus a determinism + spot-check that re-proves sampled changes (all 23 fix pushes were covered by the executed arm; the + sample re-proves the judgment lane reproduces identical judgments for identical inputs). +- Summary assembly: `arm_regression.summarize_arm` (masked repository identities, Req 5.4), + `arm_regression.four_arm_development` over `development_runner.four_arm_report` (Req 11.8), metric + headline through `benchmarks.valid.metrics` with the same scorer contract as the baseline summary + (Wilson intervals; `push.admission.defects` derived from strict-lane findings), budget result per + Req 6.1 published as development evidence (the summary itself refuses only on instrument failure). + +## Corrections during the run (instrument honesty) + +1. First probe run reported all-zero coverage in 0.0 s: the probe had skipped the sparse-checkout + disable step, so worktrees contained no files. Fixed by replicating the harness; the probe then read + 4,096-6,170 files per repo and found real entry points. +2. Two shard restarts: the first record shape missed the scorer contract (`instrument.hook_exit`, + `push.admission.defects`); 35 stale-shape records were deleted rather than patched -- back-filling + instrument fields into written records would fabricate proof. +3. `tests/test_admission_scoring.py` imported `benchmarks` without the `sys.path.insert` its sibling + test files have; it had only ever passed when an earlier-collected file fixed the path first. Fixed + with the sibling pattern. + +4. Review round 1 rejected the first summary for a false baseline provenance stamp + (`bundle_from` stamps the working-tree HEAD, so the baseline bundle claimed revision `9c95f842` instead + of `c5341d7`), for a missing covered/uncovered recall split, and for shard-scoped population digests. + Remediated by rebuilding both bundle manifests with explicit revisions and honest lane configurations, + re-adapting the baseline records from the corrected manifest, publishing the authoritative + full-population digest (`999594274a4e`, asserted in assembly) with a per-record stamp disclosure, and + adding the recall split. No arm was re-executed; no written record was back-filled. + +## Result + +All numbers from `summary.json` (this directory); raw records under `~/ousast-results/prover/ac-four-arm-1/`. +**Development regression evidence only.** + +| metric | immutable 2026-10-28 baseline | candidate arm 2026-10-30 | +|---|---|---| +| recall (fix pushes) | 15/23 = 65.2% (Wilson 44.9-81.2) | 8/23 = 34.8% (Wilson 18.8-55.1) | +| covered-repo recall | 15/21 | 8/21 = 38.1% (Wilson 20.8-59.1) | +| uncovered-repo recall | 0/2 | 0/2 (separately reported, Req 7.3) | +| location recall | 55/93 = 59.1% (Wilson 49.0-68.6) | 25/93 = 26.9% (Wilson 18.9-36.7) | +| false alarms / 100 pushes | 69 on 460 = 15.0 (Wilson 12.0-18.6) | 24 on 460 = 5.2 (Wilson 3.5-7.6) | +| precision | 15/84 = 17.9% | 8/32 = 25.0% | +| instrument failures | 0 | 0 | +| per-change seconds | mean 27.9, max 1063 | mean 17.4, median 1.1, max 240 | +| budget gate (120 s) | exceeded (development evidence) | failed: 20 changes over, max 240 s | +| uncovered repositories | 2 | 2 (masked: `1ca62ca671af`, `c92157c966cc`) | + +Four-arm attribution (Req 11.8): fact-only lift **0 on every judgment kind** -- the fact inputs are +byte-identical (digest `e7db4031d3d5`), so the entire movement is analyzer-attributed. Pair-level +totals: unguarded 340 -> 94 (-246), discharged 130 -> 0 (-130), indeterminate 0 -> 102 (+102). +Per-change transitions (baseline binary flag -> strict three-way): flagged->unguarded 29, +flagged->indeterminate 25, flagged->none 30 (84 baseline flags reconciled exactly), quiet->unguarded 3, +quiet->indeterminate 8, quiet->none 388. `discharged` is 0 across the whole strict run: on this +population the strict lane never proves a complete-path discharge -- every judged pair is unguarded or +indeterminate. + +Reading: the amended analyzer buys judgment honesty at a large recall cost on this development +population. It retracts 55 of 84 binary alarms (25 to indeterminate -- incomplete evidence, visible; +30 to no judged pairs at all), finds 3 new ones, and cuts false alarms roughly threefold +(15.0 -> 5.2 per 100). The stage funnel names the bottleneck: of 4,943,304 recorded call resolutions, +only 589,520 (11.9%) are complete; 1,451,325 are incomplete and 2,908,559 unresolved. Exact, finite +call resolution (task 8.3) starves pair formation upstream of judgment; the strict dominance and +witness rules then never reach a definite discharge. This mirrors the pre-registered risk +("indeterminate inflation ... report the stage funnel and reason distribution; improve the dominant +reason rather than weakening the lattice", research.md) and the 2026-10-28 ablation's levers +(call resolution first). Development-only conclusion for the next amendment cycle: grow entry facts +and call-resolution coverage; the three-way lane itself behaves exactly as specified -- no pair is +ever silently clean, and every refusal is visible in the funnel. + +Runtime: the executed arm ran as two resumable shards, 64.2 + 73.9 minutes wall (4 cores, 2 workers), +483/483 records, every record with nonzero-read proof (`hook_bytes_read` > 0, no zero-byte entries, +`hook_exit` 0). Determinism spot-check: 6 sampled changes (3 fix pushes, 3 from the largest +repositories) re-proved to identical judgments, findings and status. + +## Reproduction + +- Bundles: `/tmp/opencode/bundles_133.py` (fact manifests, identity proof, materialisation through + `benchmarks.valid.run_manifests`), corrected by `/tmp/opencode/fix_bundles_133.py` (explicit revisions). +- Executed arm: `/tmp/opencode/run_shard_133.py ` (shardable, resumable; module: + `benchmarks.valid.arm_regression.run_arm`, strict judgment lane). +- Baseline adaptation: `/tmp/opencode/adapt_bab_133.py` (`adapt_baseline_record`, binary vocabulary). +- Determinism: `/tmp/opencode/determinism_133.py`. +- Summary: `/tmp/opencode/assemble_133.py` (`summarize_arm`, `four_arm_development`, scorer-compatible + metrics via `benchmarks.valid.metrics` and `benchmarks.unseen.score`). diff --git a/benchmarks/measurements/2026-10-30-access-control-four-arm/summary.json b/benchmarks/measurements/2026-10-30-access-control-four-arm/summary.json new file mode 100644 index 0000000..dabfaeb --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-four-arm/summary.json @@ -0,0 +1,489 @@ +{ + "arm_execution": { + "baseline-analyzer/baseline-facts": "immutable 2026-10-28 records, adapted (vocabulary: binary)", + "baseline-analyzer/candidate-facts": "alias of baseline arm by proven input identity", + "candidate-analyzer/baseline-facts": "alias of candidate arm by proven input identity", + "candidate-analyzer/candidate-facts": "executed 2026-10-30 (vocabulary: three-way)" + }, + "bundles": { + "baseline": { + "acquisition_artifact": null, + "acquisition_record_digest": "none", + "analyzer_revision": "c5341d79a21bfae49ee157bdac19f967754f0be4", + "configuration_digest": "bc4d444e2bc4", + "fact_artifact": "facts-baseline.json", + "fact_set_digest": "3eb5bf699434" + }, + "candidate": { + "acquisition_artifact": null, + "acquisition_record_digest": "none", + "analyzer_revision": "9c95f842f7263710ec7d106794b37eb94a45425d", + "configuration_digest": "be6d0f688598", + "fact_artifact": "facts-candidate.json", + "fact_set_digest": "3eb5bf699434" + } + }, + "candidate_arm": { + "budget": { + "budget_seconds": 120.0, + "gate": "failed", + "overruns": [ + { + "id": "099364c093061a7d02d2e728a503b753441bd455ae6ae5f6970326d1815ee6f4", + "seconds": 160.65696239499812 + }, + { + "id": "1d77199cf964d7f168f3f0c157646f52aef7e3931b8da2f9fa829e1e32931964", + "seconds": 237.99327803600318 + }, + { + "id": "3a66c58b6c2b345f622f00820bd833932c5fa02ccc0d03626ac95f9b179baf55", + "seconds": 160.20379009899625 + }, + { + "id": "514850869e2c19bfc55dbc75b2c1e1766249e12f04385b1de7b4d116b6b209fa", + "seconds": 239.97186829699785 + }, + { + "id": "58a2ffe30e0bc6a166a01fe016ebf43b0c0f3385b0f4a2e8578d3e039371c17e", + "seconds": 135.12179498800106 + }, + { + "id": "59327d70f193338a3ae990177f6fa8aece8e6913e6e5d3a27888d383edb56165", + "seconds": 195.88553729499836 + }, + { + "id": "5c73c7dac092be000911ad6d4327156dbdc7a02d3d22545cfe47a25b67ce76ec", + "seconds": 201.59726526400482 + }, + { + "id": "5c9a822532e45241cb40e12ba6437837bf0301a6b2bead1261fbaa90e028f211", + "seconds": 238.12383715200122 + }, + { + "id": "720017c40d9dd8ad541f00c2577b2d1676ba3dfcaa997166a1065dbb36b42231", + "seconds": 136.7938399220002 + }, + { + "id": "75f92ee98982d750540239481ee8aa4258ec3cc3cb64cce6ca19835d9fcf2d99", + "seconds": 195.32054610600608 + }, + { + "id": "a505c341e4c5e363b3050d1523bd8f6868c10e3cd5ed383d3c8e155b49bf9a70", + "seconds": 168.9383434449992 + }, + { + "id": "bada9b8aaceaff26178d1940f7f6032b24d213071c54c30d2e7ce929e557df90", + "seconds": 170.55653589800204 + }, + { + "id": "c0fca5070cee86d26832638230b364f45d245c6e1a77333d38ee393832291e13", + "seconds": 238.88118421599938 + }, + { + "id": "c1859d8c2376e25dd46024b336bb8106a702f8139437934cf48d282cf0f396cb", + "seconds": 203.85092047799844 + }, + { + "id": "c2c52c1bb6b1f0d70b29b7b64009d6ca85c6e4d3659a195a904d577f6b02c886", + "seconds": 193.02569297700393 + }, + { + "id": "da89557ec3cada71aed38482d8269959d56e9a37877f66d69cf5b3905d0cbcff", + "seconds": 238.55912208399968 + }, + { + "id": "e0b9c856901ff36c6e54a0a73752b06f6a9996930702c18593c25936709d4c79", + "seconds": 162.15434529800405 + }, + { + "id": "e934b0222e338b768bfc594668434850f18e3a7ee35a76fb35c19898d4924349", + "seconds": 162.4909854649959 + }, + { + "id": "ea9ed8aad6259c9ea8ee1f8227a981710e8b4b1da838e74d6e68075455e78631", + "seconds": 163.5947257539956 + }, + { + "id": "f664ff98347b54ace35a50a499c77cfe3c6e06158e78f76f20b9d9317a29bb94", + "seconds": 160.8364557419991 + } + ] + }, + "development_only": true, + "instrument_failures": 0, + "judgment_counts": { + "discharged": 0, + "indeterminate": 102, + "unguarded": 94 + }, + "per_repository": { + "03d0eff98aa6": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 44, + "status": "covered", + "unguarded": 2 + }, + "0bdd00190b32": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 6179, + "status": "covered", + "unguarded": 3 + }, + "1ca62ca671af": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 0, + "status": "uncovered", + "unguarded": 0 + }, + "34b17c3454d8": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 828, + "status": "covered", + "unguarded": 13 + }, + "3caf85f05a65": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 559, + "status": "covered", + "unguarded": 23 + }, + "4b7075e3d5a1": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 93, + "status": "covered", + "unguarded": 11 + }, + "4dde9ea84248": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 100, + "status": "covered", + "unguarded": 3 + }, + "4e75cb1165d3": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 65, + "status": "covered", + "unguarded": 3 + }, + "69f3c5cdd90a": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 115, + "status": "covered", + "unguarded": 7 + }, + "6a4eac96c6ae": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 108, + "status": "covered", + "unguarded": 4 + }, + "6c1d234f9e23": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 21, + "status": "covered", + "unguarded": 0 + }, + "6c92cca94d4b": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 470, + "status": "covered", + "unguarded": 2 + }, + "760591cb57ba": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 87, + "status": "covered", + "unguarded": 5 + }, + "78d80100d68f": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 98, + "status": "covered", + "unguarded": 1 + }, + "89c10adb0673": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 91, + "status": "covered", + "unguarded": 0 + }, + "9047b2efc25b": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 86, + "status": "covered", + "unguarded": 0 + }, + "ba37e9ee7b7b": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 163, + "status": "covered", + "unguarded": 1 + }, + "c5bd586fae3f": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 33, + "status": "covered", + "unguarded": 6 + }, + "c690d767b023": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 29, + "status": "covered", + "unguarded": 5 + }, + "c92157c966cc": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 0, + "status": "uncovered", + "unguarded": 0 + }, + "d8eda62c6cc2": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 58, + "status": "covered", + "unguarded": 5 + }, + "e89a8c0f281c": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 375, + "status": "covered", + "unguarded": 0 + }, + "ed1f033972e0": { + "changes": 21, + "discharged": 0, + "frameworks_recognised": 355, + "status": "covered", + "unguarded": 0 + } + }, + "population_role": "development", + "seconds_per_change": { + "maximum": 239.97186829699785, + "mean": 17.395369377060025, + "median": 1.0545286380001926, + "n": 483 + }, + "total_changes": 483, + "uncovered_repositories": [ + "1ca62ca671af", + "c92157c966cc" + ] + }, + "comparison_to_immutable_baseline": { + "current": { + "covered_recall": { + "count": 8, + "n": 21, + "rate": 0.38095238095238093, + "wilson95": [ + 0.20750767797982506, + 0.5912170418449848 + ] + }, + "false_alarms_per_100_pushes": { + "cluster95": [ + 2.608695652173913, + 8.478260869565217 + ], + "count": 24, + "icc": 0.06521124844323971, + "n": 460, + "rate": 5.217391304347826, + "wilson95": [ + 3.5308056950422455, + 7.645768376796072 + ] + }, + "instrument_failures": 0, + "location_recall": { + "count": 25, + "n": 93, + "rate": 0.26881720430107525, + "wilson95": [ + 0.1892118216186081, + 0.36676412260370955 + ] + }, + "precision": { + "count": 8, + "n": 32, + "rate": 0.25, + "wilson95": [ + 0.1325224398262155, + 0.42106891770246624 + ] + }, + "recall_on_fix_changes": { + "count": 8, + "n": 23, + "rate": 0.34782608695652173, + "wilson95": [ + 0.18811054636982238, + 0.5511002309310836 + ] + }, + "uncovered_recall": { + "count": 0, + "n": 2, + "rate": 0.0, + "wilson95": [ + 0, + 0.6576280471103807 + ] + } + }, + "immutable_baseline": { + "false_alarms_per_100_pushes": { + "cluster95": [ + 8.695652173913043, + 21.521739130434785 + ], + "count": 69, + "icc": 0.144640754781103, + "n": 460, + "rate": 15.0, + "wilson95": [ + 12.027398310244092, + 18.552351328425665 + ] + }, + "instrument_failures": 0, + "location_recall": { + "count": 55, + "n": 93, + "rate": 0.5913978494623656, + "wilson95": [ + 0.4897977403042717, + 0.6857466538403933 + ] + }, + "precision": { + "count": 15, + "n": 84, + "rate": 0.17857142857142858, + "wilson95": [ + 0.1113109090339025, + 0.2739461901081896 + ] + }, + "recall_on_fix_changes": { + "count": 15, + "n": 23, + "rate": 0.6521739130434783, + "wilson95": [ + 0.4488997690689164, + 0.8118894536301776 + ] + } + }, + "note": "regression evidence against the immutable 2026-10-28 baseline; development population only, never an untouched-population result" + }, + "date": "2026-10-30", + "development_only": true, + "fact_inputs_identical": { + "fact_set_digest": "e7db4031d3d5", + "identical": true + }, + "four_arm": { + "arms": [ + "baseline-analyzer/baseline-facts", + "baseline-analyzer/candidate-facts", + "candidate-analyzer/baseline-facts", + "candidate-analyzer/candidate-facts" + ], + "attribution": { + "analyzer_only_lift": { + "discharged": -130, + "indeterminate": 102, + "unguarded": -246 + }, + "arms": [ + "baseline-analyzer/baseline-facts", + "baseline-analyzer/candidate-facts", + "candidate-analyzer/baseline-facts", + "candidate-analyzer/candidate-facts" + ], + "fact_only_lift": { + "discharged": 0, + "indeterminate": 0, + "unguarded": 0 + }, + "funnel_delta_fact_only": { + "seconds": 0.0 + }, + "judgment_delta_analyzer_only": { + "discharged": -130, + "indeterminate": 102, + "unguarded": -246 + }, + "judgment_delta_fact_only": { + "discharged": 0, + "indeterminate": 0, + "unguarded": 0 + }, + "transitions": [ + "flagged->indeterminate", + "flagged->none", + "flagged->unguarded", + "quiet->indeterminate", + "quiet->none", + "quiet->unguarded", + "same" + ] + }, + "fact_inputs_identical": true, + "per_change_transitions": { + "flagged->indeterminate": 25, + "flagged->none": 30, + "flagged->unguarded": 29, + "quiet->indeterminate": 8, + "quiet->none": 388, + "quiet->unguarded": 3 + } + }, + "population": { + "changes": 483, + "freeze_verified": true, + "per_record_stamp_note": "executed-arm records carry the shard-scoped population digest of the subset their worker processed (8b7be14b70df shard 0, 27fb15c10491 shard 1); this field is the authoritative full-population digest over all 483 change ids. Baseline-arm adapted records carry no population stamp from the immutable 2026-10-28 run; disclosed rather than back-filled.", + "pool": "benchmarks/valid/train/pool-train.toml", + "population_digest": "999594274a4e" + }, + "population_role": "development", + "stage_funnel": { + "entry_points": 17046, + "judgment_counts": { + "discharged": 0, + "indeterminate": 102, + "unguarded": 94 + }, + "resolution": { + "complete_calls": 589520, + "incomplete_calls": 1451325, + "unresolved_calls": 2908559 + } + } +} diff --git a/benchmarks/valid/arm_regression.py b/benchmarks/valid/arm_regression.py new file mode 100644 index 0000000..7da0e41 --- /dev/null +++ b/benchmarks/valid/arm_regression.py @@ -0,0 +1,335 @@ +"""Four-arm development regression runner (Task 13.3, Reqs 5.1-5.7, 6.1, 7.3, 11.8, 12.9). + +Runs the registered arms over the immutable development population with the +prover's strict judgment lane, proves nonzero input reads per change, writes +one resumable record per change carrying analyzer, fact-set, population and +input digests, adapts the immutable 2026-10-28 baseline records under an +explicit binary-vocabulary marker, registers cross arms only on digest-proven +input identity, masks repository identities in shared summaries, and publishes +budget results as development evidence without any admission claim. +""" + +from __future__ import annotations + +import hashlib +import json +import statistics +import subprocess +import tempfile +import time +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from benchmarks.unseen.extract import InstrumentFailure +from benchmarks.unseen.replay import canonical +from benchmarks.valid.catalog import local_git +from benchmarks.valid.development_runner import AnalyzerBundle, stage_funnel +from benchmarks.valid.prover_arm import _write_json +from openultrasast.hunt.producers import Context, _Absent, _safe_path +from openultrasast.hunt.prover import prove_change + +BundleDigests = AnalyzerBundle +BUDGET_SECONDS = 120.0 + + +class FactIdentityMismatch(ValueError): + """A cross arm was registered without digest-proven input identity.""" + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def fact_manifest_disk(directory: Path) -> dict[str, str]: + """Digest every TOML fact artifact under ``directory`` as it exists on disk.""" + manifest: dict[str, str] = {} + for path in sorted(directory.rglob("*.toml")): + if path.is_file(): + manifest[path.relative_to(directory).as_posix()] = _sha256_bytes(path.read_bytes()) + return manifest + + +def fact_manifest_ref(repo_root: Path, ref: str, subpath: str = "") -> dict[str, str]: + """Digest the content of every TOML fact artifact recorded at ``ref`` under ``subpath``.""" + listing = subprocess.run( + ["git", "-C", str(repo_root), "ls-tree", "-r", "--name-only", ref, "--", subpath], + capture_output=True, + text=True, + check=True, + ).stdout.splitlines() + prefix = f"{subpath}/" if subpath else "" + manifest: dict[str, str] = {} + for path in listing: + if not path.endswith(".toml"): + continue + blob = subprocess.run(["git", "-C", str(repo_root), "show", f"{ref}:{path}"], capture_output=True, check=True).stdout + manifest[path.removeprefix(prefix)] = _sha256_bytes(blob) + return manifest + + +def manifest_digest(manifest: dict[str, str]) -> str: + return hashlib.sha256(json.dumps(manifest, sort_keys=True).encode()).hexdigest()[:12] + + +def fact_identity(baseline: dict[str, str], candidate: dict[str, str]) -> dict[str, Any]: + """Register a cross-arm input identity only when the fact sets are byte-equal.""" + if baseline != candidate: + raise FactIdentityMismatch("cross arms require digest-proven identical fact inputs") + return {"identical": True, "fact_set_digest": manifest_digest(candidate)} + + +def adapt_baseline_record(record: dict[str, Any]) -> dict[str, Any]: + """Adapt one immutable 2026-10-28 record under its honest binary vocabulary.""" + findings = record.get("findings") + coverage = record.get("coverage") + if findings is None or not isinstance(coverage, dict): + raise ValueError(f"baseline record {record.get('case_id', '?')} lacks findings or coverage") + return { + "change_id": str(record.get("case_id", record.get("id", ""))), + "repo": record.get("repo", ""), + "kind": record.get("kind", ""), + "vocabulary": "binary", + "stage_funnel": { + "entry_recognition": {"entry_points": int(coverage.get("entry_points", 0))}, + "resolution": {}, + "judgment": { + "unguarded": len(findings), + "discharged": int(coverage.get("operations_discharged", 0)), + "indeterminate": 0, + }, + }, + "seconds": float(record.get("seconds", 0.0)), + } + + +def _input_digest(change: dict[str, Any], arm_name: str) -> str: + return hashlib.sha256(canonical({"change": change, "arm": arm_name})).hexdigest()[:12] + + +def _population_digest(changes: tuple[dict[str, Any], ...]) -> str: + joined = "\n".join(sorted(str(change.get("id", "")) for change in changes)) + return hashlib.sha256(joined.encode()).hexdigest()[:12] + + +def _prove_one( + bundle: BundleDigests, + arm_name: str, + population: str, + change: dict[str, Any], + repo_path: Path, +) -> dict[str, Any]: + started = time.monotonic() + instrument: dict[str, Any] = dict( + hook_exit=0, + hook_bytes_read=0, + changed_bytes={}, + file_bytes={}, + head_verified=False, + base_verified=False, + suspect_fast=False, + failure=None, + ) + record: dict[str, Any] = { + **change, + "arm": arm_name, + "analyzer_revision": bundle.analyzer_revision, + "fact_set_digest": bundle.fact_set_digest, + "acquisition_record_digest": bundle.acquisition_record_digest, + "configuration_digest": bundle.configuration_digest, + "population_digest": population, + "input_digest": _input_digest(change, arm_name), + "done": True, + "status": "prover_covered", + "vocabulary": "three-way", + "instrument": instrument, + "findings": [], + "traces": [], + "coverage": {}, + "degradations": [], + "stage_funnel": {}, + "push": {"admission": {"defects": [], "dispositions": []}, "quick_tier": []}, + } + try: + git = local_git(repo_path) + git.run("cat-file", "-e", change["base"] + "^{commit}") + instrument["base_verified"] = True + with tempfile.TemporaryDirectory(prefix="ousast-arm-") as temporary: + root = Path(temporary) / "head" + git.run("-c", "core.hooksPath=/dev/null", "worktree", "add", "--detach", str(root), change["head"]) + try: + _verify_checkout(root) + ctx = Context(root, change) + ctx.verify_head() + instrument["head_verified"] = True + _instrument_reads(ctx, root, change, instrument) + result = prove_change(root, change, ctx, judgment_mode=True) + instrument["hook_bytes_read"] = sum(instrument["file_bytes"].values()) + record.update( + findings=list(result.findings), + traces=list(result.traces), + coverage=asdict(result.coverage), + degradations=list(result.degradations), + stage_funnel=stage_funnel(result), + ) + record["push"]["admission"]["defects"] = [ + {**finding, "defect_id": finding["id"], "source": "prover", "locations": [[finding["path"], finding["line"]]]} + for finding in result.findings + ] + finally: + git.run("worktree", "remove", "--force", str(root)) + except Exception as error: # noqa: BLE001 - the record must survive any instrument failure + instrument.update(hook_exit=2, failure=type(error).__name__) + record.update(status="instrument_failure", findings=[]) + record["seconds"] = time.monotonic() - started + return record + + +def _verify_checkout(root: Path) -> None: + checked_git = local_git(root) + sparse = checked_git.runner(["git", "-C", str(root), "config", "--get", "core.sparseCheckout"], capture_output=True, text=True) + if sparse.stdout.strip() == "true": + checked_git.run("sparse-checkout", "disable") + if checked_git.run("status", "--porcelain", "--untracked-files=no").strip(): + raise InstrumentFailure("incomplete checkout") + + +def _instrument_reads(ctx: Context, root: Path, change: dict[str, Any], instrument: dict[str, Any]) -> None: + changed = {file["path"] for file in change.get("files", [])} + for relative in sorted(changed | set(ctx.product_files)): + path = _safe_path(root, relative) + if not path.is_file(): + held = local_git(root).runner(["git", "-C", str(root), "cat-file", "-e", f"HEAD:{relative}"], capture_output=True) + if held.returncode == 0: + raise InstrumentFailure("unread tracked file") + continue + try: + text = ctx.read(relative) + except _Absent as error: + # A file git holds empty is read, not unread; anything else absent is unread. + held = local_git(root).runner(["git", "-C", str(root), "cat-file", "-s", f"HEAD:{relative}"], capture_output=True, text=True) + if getattr(error, "reason", "") == "empty_source" and held.returncode == 0 and int(held.stdout.strip() or 0) == 0: + instrument["empty_files"] = instrument.get("empty_files", 0) + 1 + continue + raise InstrumentFailure("unread tracked file") from error + size = len(text.encode("utf-8")) + instrument["file_bytes"][relative] = size + if relative in changed: + instrument["changed_bytes"][relative] = size + if not size: + raise InstrumentFailure("zero-byte read") + + +def run_arm( + *, + arm_name: str, + bundle: BundleDigests, + changes: tuple[dict[str, Any], ...], + repos: dict[str, str], + out_dir: Path, +) -> tuple[dict[str, Any], ...]: + """Prove every change on the strict judgment lane; records resume by input digest.""" + out_dir.mkdir(parents=True, exist_ok=True) + population = _population_digest(changes) + records: list[dict[str, Any]] = [] + for change in changes: + path = out_dir / f"{change['id']}.json" + if path.is_file(): + previous = json.loads(path.read_text(encoding="utf-8")) + if previous.get("input_digest") == _input_digest(change, arm_name): + records.append(previous) + continue + raise ValueError(f"existing record for {change['id']} does not match this arm's inputs") + record = _prove_one(bundle, arm_name, population, change, Path(repos[change["repo"]]).expanduser()) + _write_json(path, record) + records.append(record) + return tuple(records) + + +def _masked(repo: str) -> str: + return hashlib.sha256(repo.encode()).hexdigest()[:12] + + +def summarize_arm(records: tuple[dict[str, Any], ...]) -> dict[str, Any]: + """Aggregate one arm; refuse any zero-byte read, publish budget and masked coverage.""" + judgment = {"unguarded": 0, "discharged": 0, "indeterminate": 0} + per_repository: dict[str, dict[str, Any]] = {} + for record in records: + instrument = record.get("instrument", {}) + if any(size == 0 for field in ("changed_bytes", "file_bytes") for size in instrument.get(field, {}).values()): + raise InstrumentFailure(f"zero-byte read of a tracked file; summary refused: {record['id']}") + funnel = record.get("stage_funnel", {}) + for kind, count in funnel.get("judgment", {}).items(): + judgment[kind] += int(count) + row = per_repository.setdefault( + _masked(record["repo"]), + {"changes": 0, "unguarded": 0, "discharged": 0, "frameworks_recognised": 0}, + ) + row["changes"] += 1 + for kind in ("unguarded", "discharged"): + row[kind] += int(funnel.get("judgment", {}).get(kind, 0)) + recognised = funnel.get("entry_recognition", {}).get("frameworks_recognised", {}) + row["frameworks_recognised"] += sum(recognised.values()) if isinstance(recognised, dict) else int(recognised) + for row in per_repository.values(): + row["status"] = "uncovered" if not row["frameworks_recognised"] else "covered" + uncovered = sorted(key for key, row in per_repository.items() if row["status"] == "uncovered") + seconds = [float(record.get("seconds", 0.0)) for record in records] + overruns = [ + {"id": record["id"], "seconds": float(record.get("seconds", 0.0))} + for record in records + if float(record.get("seconds", 0.0)) > BUDGET_SECONDS + ] + return { + "population_role": "development", + "development_only": True, + "total_changes": len(records), + "judgment_counts": judgment, + "budget": { + "gate": "failed" if overruns else "passed", + "budget_seconds": BUDGET_SECONDS, + "overruns": overruns, + }, + "seconds_per_change": dict( + n=len(seconds), + mean=statistics.mean(seconds) if seconds else 0.0, + median=statistics.median(seconds) if seconds else 0.0, + maximum=max(seconds, default=0.0), + ), + "instrument_failures": sum(record.get("status") == "instrument_failure" for record in records), + "per_repository": per_repository, + "uncovered_repositories": uncovered, + } + + +def four_arm_development( + *, + bab: tuple, + bacf: tuple, + cabf: tuple, + cacfon: tuple, + fact_identity: dict[str, Any], +) -> dict[str, Any]: + """Publish attribution and transitions between compared arms (Req 11.8).""" + from benchmarks.valid.development_runner import four_arm_report + + attribution = four_arm_report(bab, bacf, cabf, cacfon) + return { + "arms": attribution["arms"], + "attribution": attribution, + "fact_inputs_identical": bool(fact_identity.get("identical")), + } + + +__all__ = [ + "BUDGET_SECONDS", + "BundleDigests", + "FactIdentityMismatch", + "adapt_baseline_record", + "fact_identity", + "fact_manifest_disk", + "fact_manifest_ref", + "four_arm_development", + "manifest_digest", + "run_arm", + "summarize_arm", +] diff --git a/tests/test_admission_scoring.py b/tests/test_admission_scoring.py index fdc0007..5ef68c7 100644 --- a/tests/test_admission_scoring.py +++ b/tests/test_admission_scoring.py @@ -1,9 +1,13 @@ """Paired scoring and the admission report (Task 12.6, Reqs 5.2, 12.4, 12.5, 12.7-12.9).""" import json +import sys +from pathlib import Path import pytest +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from benchmarks.valid.admission_scoring import ( AdmissionCase, AdmissionScoringError, diff --git a/tests/test_arm_regression.py b/tests/test_arm_regression.py new file mode 100644 index 0000000..5a71ca2 --- /dev/null +++ b/tests/test_arm_regression.py @@ -0,0 +1,260 @@ +"""Task 13.3 -- four-arm development regression (Reqs 5.1-5.7, 6.1, 7.3, 11.8, 12.9). + +The regression runner proves nonzero input reads, writes resumable per-change +records with bundle digests, adapts the immutable 2026-10-28 baseline records +honestly (binary vocabulary marker), registers input-identical cross arms only +on digest proof, masks repository identities in shared summaries, and never +makes an admission claim. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from benchmarks.valid import arm_regression # noqa: E402 + + +def _git(repo: Path, *args: str) -> None: + subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True) + + +@pytest.fixture() +def sample_repo(tmp_path: Path) -> Path: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("def handler(event):\n return event\n") + (repo / "ruleset").mkdir() + (repo / "ruleset" / "facts.toml").write_text("[entry]\nids = ['a']\n") + _git(repo, "init", "-q") + _git(repo, "config", "user.email", "t@example.com") + _git(repo, "config", "user.name", "t") + _git(repo, "add", ".") + _git(repo, "commit", "-qm", "first") + return repo + + +def _change(repo: Path) -> dict[str, object]: + head = subprocess.run(["git", "-C", str(repo), "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip() + return { + "id": "sample-repo-ordinary-abc", + "case_id": "sample-repo-ordinary-abc", + "repo": "sample/repo", + "kind": "ordinary", + "base": head, + "head": head, + "method": "first_parent", + "files": [{"path": "app.py", "head_lines": [[1, 2]]}], + } + + +def _bundle() -> arm_regression.BundleDigests: + return arm_regression.BundleDigests( + analyzer_revision="f" * 12, + fact_set_digest="a" * 12, + acquisition_record_digest="none", + configuration_digest="c" * 12, + ) + + +def test_fact_manifest_ref_matches_clean_disk(sample_repo: Path) -> None: + manifest = arm_regression.fact_manifest_disk(sample_repo / "ruleset") + assert manifest == arm_regression.fact_manifest_ref(sample_repo, "HEAD", "ruleset") + + +def test_fact_manifest_detects_disk_drift(sample_repo: Path) -> None: + before = arm_regression.fact_manifest_ref(sample_repo, "HEAD", "ruleset") + (sample_repo / "ruleset" / "facts.toml").write_text("[entry]\nids = ['a', 'b']\n") + _git(sample_repo, "add", ".") + _git(sample_repo, "commit", "-qm", "drift") + assert arm_regression.manifest_digest(before) != arm_regression.manifest_digest( + arm_regression.fact_manifest_ref(sample_repo, "HEAD", "ruleset") + ) + + +def test_fact_identity_alias_requires_equal_digests() -> None: + left = {"src/ruleset/a.toml": "0" * 64} + right = {"src/ruleset/a.toml": "0" * 64} + alias = arm_regression.fact_identity(left, right) + assert alias["identical"] is True + with pytest.raises(arm_regression.FactIdentityMismatch): + arm_regression.fact_identity(left, {"src/ruleset/a.toml": "1" * 64}) + + +def test_adapt_baseline_record_maps_binary_vocabulary() -> None: + record = { + "case_id": "c1", + "repo": "a/b", + "kind": "ordinary", + "findings": [{"id": "f1"}, {"id": "f2"}], + "coverage": {"entry_points": 5, "operations_checked": 4, "operations_discharged": 1}, + "seconds": 3.5, + "instrument": {"hook_bytes_read": 100}, + } + adapted = arm_regression.adapt_baseline_record(record) + judgment = adapted["stage_funnel"]["judgment"] + assert judgment == {"unguarded": 2, "discharged": 1, "indeterminate": 0} + assert adapted["vocabulary"] == "binary" + assert adapted["seconds"] == 3.5 + + +def test_adapt_baseline_record_refuses_incomplete_record() -> None: + with pytest.raises(ValueError): + arm_regression.adapt_baseline_record({"case_id": "c1", "coverage": {}}) + + +def test_run_arm_writes_resumable_record_with_digests(sample_repo: Path, tmp_path: Path) -> None: + change = _change(sample_repo) + out = tmp_path / "arm" + records = arm_regression.run_arm( + arm_name="candidate-analyzer/candidate-facts", + bundle=_bundle(), + changes=(change,), + repos={"sample/repo": str(sample_repo)}, + out_dir=out, + ) + assert len(records) == 1 + record = records[0] + assert record["fact_set_digest"] == "a" * 12 + assert record["analyzer_revision"] == "f" * 12 + assert record["instrument"]["hook_bytes_read"] > 0 + assert record["instrument"]["base_verified"] is True + assert record["instrument"]["hook_exit"] == 0 + assert isinstance(record["push"], dict) + assert record["push"]["admission"]["defects"] == [] + assert record["method"] == "first_parent" + assert (out / "sample-repo-ordinary-abc.json").is_file() + first_mtime = (out / "sample-repo-ordinary-abc.json").stat().st_mtime_ns + arm_regression.run_arm( + arm_name="candidate-analyzer/candidate-facts", + bundle=_bundle(), + changes=(change,), + repos={"sample/repo": str(sample_repo)}, + out_dir=out, + ) + assert (out / "sample-repo-ordinary-abc.json").stat().st_mtime_ns == first_mtime + + +def test_run_arm_allows_legitimately_empty_tracked_file(sample_repo: Path, tmp_path: Path) -> None: + (sample_repo / "empty.py").write_text("") + (sample_repo / "app.py").write_text("def handler(event):\n return 2\n") + _git(sample_repo, "add", ".") + _git(sample_repo, "commit", "-qm", "empty sibling") + change = _change(sample_repo) + change["files"] = [ + {"path": "app.py", "head_lines": [[1, 2]]}, + {"path": "empty.py", "head_lines": [[1, 1]]}, + ] + records = arm_regression.run_arm( + arm_name="candidate-analyzer/candidate-facts", + bundle=_bundle(), + changes=(change,), + repos={"sample/repo": str(sample_repo)}, + out_dir=tmp_path / "arm", + ) + assert records[0]["status"] == "prover_covered" + assert records[0]["instrument"]["empty_files"] == 1 + assert records[0]["instrument"]["file_bytes"]["app.py"] > 0 + + +def test_summarize_arm_refuses_zero_byte_summary(tmp_path: Path) -> None: + record = { + "id": "c1", + "case_id": "c1", + "repo": "a/b", + "kind": "ordinary", + "status": "instrument_failure", + "instrument": {"file_bytes": {"app.py": 0}, "hook_bytes_read": 0}, + "stage_funnel": {"judgment": {"unguarded": 0, "discharged": 0, "indeterminate": 0}}, + "seconds": 1.0, + } + (tmp_path / "c1.json").write_text(json.dumps(record)) + with pytest.raises(arm_regression.InstrumentFailure): + arm_regression.summarize_arm((record,)) + + +def test_summarize_arm_publishes_budget_result_and_masks_repositories() -> None: + base = { + "id": "c1", + "case_id": "c1", + "repo": "private/name", + "kind": "ordinary", + "status": "prover_covered", + "instrument": {"file_bytes": {"app.py": 10}, "hook_bytes_read": 10}, + "stage_funnel": { + "entry_recognition": {"entry_points": 1, "frameworks_recognised": {"web": 1}}, + "resolution": {"resolved": 1}, + "judgment": {"unguarded": 0, "discharged": 1, "indeterminate": 0}, + }, + "seconds": 121.0, + "fact_set_digest": "a" * 12, + "analyzer_revision": "f" * 12, + } + summary = arm_regression.summarize_arm((base,)) + assert summary["budget"]["gate"] == "failed" + assert summary["budget"]["overruns"][0]["id"] == "c1" + assert summary["budget"]["overruns"][0]["seconds"] == 121.0 + assert "private/name" not in json.dumps(summary) + assert all(len(key) == 12 for key in summary["per_repository"]) + assert summary["population_role"] == "development" + assert summary["development_only"] is True + + +def test_four_arm_development_has_no_admission_language() -> None: + from benchmarks.valid import development_runner + + def rec(unguarded: int) -> development_runner.DevelopmentRecord: + return development_runner.DevelopmentRecord( + change_id="c1", + analyzer_revision="f" * 12, + fact_set_digest="a" * 12, + acquisition_record_digest="none", + configuration_digest="c" * 12, + population_digest="d" * 12, + input_digest="e" * 12, + stage_funnel={"judgment": {"unguarded": unguarded, "discharged": 0, "indeterminate": 0}}, + judgment_transition="same", + seconds=1.0, + ) + + report = arm_regression.four_arm_development( + bab=(rec(1),), + bacf=(rec(0),), + cabf=(rec(0),), + cacfon=(rec(0),), + fact_identity={"identical": True}, + ) + text = repr(report) + for word in ("admission", "vulnerable", "ordinary", "label"): + assert word not in text.lower() + assert report["fact_inputs_identical"] is True + assert report["attribution"]["judgment_delta_analyzer_only"]["unguarded"] == -1 + + +def test_summarize_arm_reports_uncovered_repository_separately() -> None: + record = { + "id": "c2", + "case_id": "c2", + "repo": "hidden/other", + "kind": "vulnerability", + "status": "prover_covered", + "instrument": {"file_bytes": {"app.py": 10}, "hook_bytes_read": 10}, + "stage_funnel": { + "entry_recognition": {"entry_points": 0, "frameworks_recognised": {}}, + "resolution": {}, + "judgment": {"unguarded": 0, "discharged": 0, "indeterminate": 0}, + }, + "seconds": 2.0, + "fact_set_digest": "a" * 12, + "analyzer_revision": "f" * 12, + } + summary = arm_regression.summarize_arm((record,)) + masked = list(summary["per_repository"])[0] + assert summary["per_repository"][masked]["status"] == "uncovered" + assert summary["uncovered_repositories"] == [masked]