diff --git a/.kiro/specs/detection-core/tasks.md b/.kiro/specs/detection-core/tasks.md index 9842552..1092ccb 100644 --- a/.kiro/specs/detection-core/tasks.md +++ b/.kiro/specs/detection-core/tasks.md @@ -147,6 +147,7 @@ _Requirement IDs use N.M where N is the requirement number in requirements.md an - _Requirements: 10.1, 10.2, 7.3_ - _Boundary: fact-pack contract + benchmark_ - _Depends: 6.1, 4.2_ + - _Parked 2026-10-31: final gated round NO-GO #3 (rule 10 stop condition, maintainer strategy-review); done-when unreachable for this arm -- the unique-declaration member-call binding increment ships (dc28325, dev complete-share 11.9->19.2 with recall/FA unchanged) but the admission gap is entry recognition (4/12 repos unrecognised; coverage 8/12) and framework operation facts, not call resolution. See benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/_ - [ ] 8.3 Pre-push hook consumes the core consumer contract - pre-push-safety-net builds PushRequest and consumes PushVerdict via core/consumer.py; budget from the measured per-class derivation; one real dev-pool push E2E. - Done when an E2E hook run on a dev-pool repo emits a PushVerdict with witnesses/reasons and a measured-class budget. diff --git a/.kiro/steering/detection-quality.md b/.kiro/steering/detection-quality.md index b7085cc..90274a9 100644 --- a/.kiro/steering/detection-quality.md +++ b/.kiro/steering/detection-quality.md @@ -73,3 +73,18 @@ and authorized **exactly one further gated round on operation facts and call res for the recognized entry shapes**. This is the final incremental round for this arm: a third admission NO-GO parks it permanently regardless of diagnosis, and the rule-11 fallback (narrowed one-ecosystem advisory product plus proof-based BLOCK) applies. + +### Final gated round outcome (2026-10-31): ARM PARKED + +The final round ran its full course (`benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/`): +the authorized resolution lever (unique-declaration member-call binding, dc28325) moved the +dev-pool funnel (complete-calls share 11.9% -> 19.2%) with dev recall and false alarms exactly +unchanged, but is disconnected from the admission population -- zero unique-name member calls +exist across all 12 admission repositories. Admission run ac-admission-3 reproduced run 2 +exactly: 0/12 pairs, third NO-GO on the same pre-registered criteria. The stop condition +fired: **the access-control arm is parked permanently**; the rule-11 fallback (narrowed +one-ecosystem advisory product plus proof-based BLOCK) applies. The binding itself ships -- +it is a strict improvement where repository-defined names exist. Diagnosis for any future +strategy review: the remaining admission gap is entry recognition (4/12 repositories with +zero frameworks recognised; coverage 8/12) and operation facts for framework-defined +operations, not call resolution. diff --git a/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/NOTES.md b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/NOTES.md new file mode 100644 index 0000000..0c446ff --- /dev/null +++ b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/NOTES.md @@ -0,0 +1,135 @@ +# 2026-10-31 access-control admission: unique-declaration member-call binding (final gated round) + +Round authorization: maintainer strategy-review 2026-10-31 (commit 1481bb3) -- exactly one +further gated round on operation facts and call resolution; a third admission NO-GO parks the +arm permanently and the rule-11 fallback applies. + +Lever (rule 12): resolution -- unique-declaration member-call binding in +`src/openultrasast/semantic/obligations/resolution.py` (commit dc28325): `x.method()` binds +complete iff exactly one same-name method declaration exists repository-wide; collisions stay +finite candidate sets; module-level functions never bind. RED->GREEN with reviewer approval; +keyword regression 856 passed (selection: obligation/prover/symbol/call_resolution/... entry/ +core/handler union). + +## Measurement 1: in-family site funnel (before/after, frozen ac-admission-2 records) + +Re-proved the three recognized (in-family) vulnerable sites with the binding at dc28325, +same changes, same repos, same facts: + +| site | complete | incomplete | unresolved (before -> after) | +|---|---|---|---| +| matrix-org/matrix-appservice-bridge 3a80 | 1 | 0 | 75 -> 75 | +| fastapi-users/fastapi-users 178f | 15 | 0 | 286 -> 286 | +| bytebase/dbhub f262 | 160 | 0 | 1173 -> 1173 | + +Zero movement on every coverage and judgment field. Instrument proof per site: fresh runs, +`hook_bytes_read` 301065 / 100846 / 677782 (nonzero), head+base verified, no failures. + +Root cause (classifier replicating the prove pipeline over each whole product tree): + +- matrix: variable-receiver calls with exactly-one same-name method = 0 (methods_named=0: 687) +- fastapi-users: 0 (methods_named=0: 146) +- dbhub: 0 (methods_named=0: 1182) +- dominant unresolved mass: member calls to framework/stdlib names with no repository + declaration (variable:methods_named=0 total 2015) and bare calls whose name has no + declaration in the calling module (207/134/818) -- imported or framework names. + +Conclusion: the authorized resolution lever is disconnected from the recognized admission +sites. The unresolved mass is a recognition/facts problem (callee names absent from the +repository), not a member-call binding problem. + +## Screen: whole admission population (exact) + +Re-proved all 264 admission records at dc28325 (fresh instrument run per record, nonzero +`hook_bytes_read` asserted, 0 zero-reads): + +- vulnerable: 0/12 records changed +- fixed: 0/12 records changed +- ordinary: 0/240 records changed + +Therefore the formal admission run with the binding is byte-determined to reproduce run 2's +report (numbers identical; only the stamped analyzer revision differs). Measurement 3 executes +that run for the record. + +(Fixed-side screen note: the first fixed-side screen attempt double-applied +`_fixed_side_change` to already-swapped frozen records -- proving vulnerable trees -- and was +discarded; the corrected screen strips stamped fields and proves the frozen fixed changes +as-is. Control re-proves at pre-binding 1481bb3 confirmed the drift was the reconstruction +bug, not the binding.) + +## Screen: dev pool (23 repos, classifier at HEAD worktrees) + +The binding fires 23,112 times across the dev-pool repositories +(variable-receiver calls with exactly one same-name method; largest: apache/seatunnel 14,198, +kimai/kimai 3,182, shopperlabs/shopper 1,296). The dev funnel moves; measurement 2 is live. + +(Two earlier dev-screen attempts were discarded as failed instruments under rule 8: the hist +clones are bare and carry a sparse-checkout config, so passing them directly -- or worktree +add without `_verify_checkout` -- reads zero bytes and silently reports plausible empty +buckets. The discarded runs produced no numbers; the screen above proves nonzero bytes_read +per repository.) + +## Measurement 2: dev-pool arm re-run + +Bundle: HEAD code dc28325 + the 13.3 candidate facts (digest 3eb5bf699434, NO express entry +pack -- the pack is a run-2 admission-gated delta, excluded here so the dev comparison +isolates exactly the binding; the only fact delta vs HEAD is entries/express.json, verified). +Configuration digest be6d0f688598 == 13.3. Output: ~/ousast-results/prover/ac-four-arm-2/cacfon. + +RESULTS (483/483 records, 2 shards, 67.2 + 76.3 min, 0 instrument failures, all 483 records +with nonzero instrument.hook_bytes_read): + +- Lever's number: complete-calls share 11.9% -> **19.2%** (589,520 -> 949,998 complete; + incomplete 1,451,325 -> 1,090,847 -- an exact 360,478-call conservation, unresolved + unchanged 2,908,559; 441/483 records moved resolution counts). +- Gates (must not regress vs 13.3): recall on fix changes 8/23 = 34.8% (identical); + location recall 25/93 = 26.9% (identical); false alarms 5.2/100 pushes (identical, + 24 flags on 460 ordinary in both arms); unguarded judgments 32 -> 32; instrument + failures 0 -> 0. +- Judgment movement: 5 ordinary records none -> indeterminate (jenkinsci/database-plugin x2, + jenkinsci/dependency-track-plugin x2, kimai/kimai x1); indeterminate 33 -> 38. No new + alarms, no recall movement; the moved records are visible bounded silence, not flags. + +Verdict: PASS -- the resolution lever moves the dev funnel materially without regressing +dev recall or false alarms. + +## Measurement 3: fresh freeze + admission run ac-admission-3 + +Freeze: facts == HEAD == run 2 facts (digest equality asserted), population pool-valid +(freeze guard 0), criteria byte-identical to runs 1-2 (asserted), analyzer dc28325. +Output: ~/ousast-results/prover/ac-admission-3. + +RESULTS (264/264 records fresh: all stamped dc28325, all nonzero hook reads, none stale; +report 2026-10-31-access-control-admission-unique-binding/report.json): + +- decision NO-GO; failed gates identical to run 2: min_fix_recall 0.0 < 0.5, + min_pair_correctness 0/12 < 0.5, min_precision 0.0 < 0.25, min_verified_location_recall + not measured (n=0); instrument failures 0. +- Per-repo comparison vs ac-admission-2: all 12 repositories identical on recognition, + unguarded, indeterminate and findings (same table as the screen predicted). +- Elapsed 260 s for the full population. + +## Round verdict + +Third admission NO-GO on the same pre-registered criteria. Under the maintainer's +2026-10-31 strategy-review decision (recorded in .kiro/steering/detection-quality.md) the +access-control arm is PARKED PERMANENTLY: no further incremental rounds. The rule-11 +fallback applies -- narrowed one-ecosystem advisory product (option C) plus proof-based +BLOCK for demonstrated findings. + +What the round established, for the parking record: + +1. The authorized resolution lever is real where repository-defined names exist: dev-pool + complete-calls share 11.9% -> 19.2% with dev recall/FA exactly unchanged + (measurement 2). The binding ships (commit dc28325). +2. The lever is disconnected from the admission population: zero member calls with a + unique repository declaration exist across all 12 admission repositories (screens + + ac-admission-3 identical to run 2). The admission funnel dies on framework callee names + with no repository declaration and on unrecognized entries -- a facts/recognition + problem that the authorized levers could not reach, confirmed by three independent + instruments (site classifier, whole-population re-prove screen, formal admission run). +3. Diagnosis for any future strategy review: the remaining admission gap is NOT call + resolution. It is (a) entry recognition on 4/12 repositories (zero frameworks + recognised; coverage 8/12) and (b) operation facts for framework-defined operations + (orm-read exists but the checked-operation count is 0 everywhere because entries are + unreached or frameworks unrecognised). diff --git a/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/bundle/facts-admission.json b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/bundle/facts-admission.json new file mode 100644 index 0000000..6e8eded --- /dev/null +++ b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/bundle/facts-admission.json @@ -0,0 +1,21 @@ +{ + "c/rules.toml": "88ed02da9fef6f170b912f601df12d3040b990d962b28ce4021f3f335442f713", + "entries/express.json": "e8e149309abd78f24206c58fde7f73ae2f60364d8b8fee22ecaee8342798d8ce", + "families.toml": "5b4c0487c755e8a280e33bbcfb1f497dc40d8703b5961c6f9322a17a3291881a", + "frameworks.toml": "cbb9f791e25d73cc215315b2dbfe9654d2ccd5747852e2a0d11ee45ea3012642", + "groovy/rules.toml": "ed0ab48e2da2ad9d084a2e5f55ac929e94ea381000d674c160843f79c352396a", + "java/rules.toml": "5934e10f595f907f37629d3b05cb56744f70d3c5cb534231b4b3f13a6510c234", + "javascript/rules.toml": "075ea7439d65082ecacdf9d9b98397ee278ed93c11ebf552687603d8fb171ce3", + "obligations/java.toml": "9125cb707312a72ac766cd3a10e1a09063ac4a2861487ebcb342eadb620e9108", + "obligations/javascript.toml": "14670c32a3622b213eb553256a7ffb38c40ffb3398012eb6f02a1e4da04d9a33", + "obligations/php.toml": "a7bcbd9bedace740567c5900db91acaa1d0aeb2049dea3101f518acb934827ce", + "obligations/python.toml": "f65cb000dbf9e777dce0f2888b0dca39b531b1624dcc3026cc2d55461647b983", + "obligations/typescript.toml": "5b9a9bf1bb4c1846a518f724db161b4b5aa533451bc32aaf84ecde851d824bdf", + "php/rules.toml": "e839846d760b57c7d65e4fcad6c206e1f1aa852d0ec4bbb9b0b459e761467115", + "python/rules.toml": "e323456fd46b5c3e4092a92463695458a620830b6d9b5f05a4ade1e14d78a5c9", + "semantic/c.toml": "2d4bbc842596fc17a70e51e528b57af5961ef05515259a57a203faa18018fa4e", + "semantic/java.toml": "e7f5b73ffc0fe71d7a9096bdf705ee8cc8e5659e2715cdb1705bd59f6c31dced", + "semantic/javascript.toml": "91c6aa895021aff755c4bf4b4ed4f4c7d8a991d260f1f9d9363be68f2134dfd5", + "semantic/php.toml": "8c7585c1241bd904558389f66decf88c27554107fc7c6e3fa1b0855797e8ea0c", + "semantic/python.toml": "97fd7cfd79ef90a3d51255d951c9f4e1ccabe09d5b6b2574f4ff788556977ab0" +} diff --git a/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/bundle/manifest.json b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/bundle/manifest.json new file mode 100644 index 0000000..3f98d14 --- /dev/null +++ b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/bundle/manifest.json @@ -0,0 +1,8 @@ +{ + "acquisition_artifact": null, + "acquisition_record_digest": "none", + "analyzer_revision": "dc2832564ae623d0ab558381e7dad4cd17c951b6", + "configuration_digest": "06e9304ee277", + "fact_artifact": "facts-admission.json", + "fact_set_digest": "124871987a06" +} diff --git a/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/criteria.json b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/criteria.json new file mode 100644 index 0000000..6c5bb35 --- /dev/null +++ b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/criteria.json @@ -0,0 +1,8 @@ +{ + "max_flagged_ordinary_per_100": 10.0, + "max_indeterminate_rate": 0.5, + "min_fix_recall": 0.5, + "min_pair_correctness": 0.5, + "min_precision": 0.25, + "min_verified_location_recall": 0.5 +} diff --git a/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/facts-admission.json b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/facts-admission.json new file mode 100644 index 0000000..6e8eded --- /dev/null +++ b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/facts-admission.json @@ -0,0 +1,21 @@ +{ + "c/rules.toml": "88ed02da9fef6f170b912f601df12d3040b990d962b28ce4021f3f335442f713", + "entries/express.json": "e8e149309abd78f24206c58fde7f73ae2f60364d8b8fee22ecaee8342798d8ce", + "families.toml": "5b4c0487c755e8a280e33bbcfb1f497dc40d8703b5961c6f9322a17a3291881a", + "frameworks.toml": "cbb9f791e25d73cc215315b2dbfe9654d2ccd5747852e2a0d11ee45ea3012642", + "groovy/rules.toml": "ed0ab48e2da2ad9d084a2e5f55ac929e94ea381000d674c160843f79c352396a", + "java/rules.toml": "5934e10f595f907f37629d3b05cb56744f70d3c5cb534231b4b3f13a6510c234", + "javascript/rules.toml": "075ea7439d65082ecacdf9d9b98397ee278ed93c11ebf552687603d8fb171ce3", + "obligations/java.toml": "9125cb707312a72ac766cd3a10e1a09063ac4a2861487ebcb342eadb620e9108", + "obligations/javascript.toml": "14670c32a3622b213eb553256a7ffb38c40ffb3398012eb6f02a1e4da04d9a33", + "obligations/php.toml": "a7bcbd9bedace740567c5900db91acaa1d0aeb2049dea3101f518acb934827ce", + "obligations/python.toml": "f65cb000dbf9e777dce0f2888b0dca39b531b1624dcc3026cc2d55461647b983", + "obligations/typescript.toml": "5b9a9bf1bb4c1846a518f724db161b4b5aa533451bc32aaf84ecde851d824bdf", + "php/rules.toml": "e839846d760b57c7d65e4fcad6c206e1f1aa852d0ec4bbb9b0b459e761467115", + "python/rules.toml": "e323456fd46b5c3e4092a92463695458a620830b6d9b5f05a4ade1e14d78a5c9", + "semantic/c.toml": "2d4bbc842596fc17a70e51e528b57af5961ef05515259a57a203faa18018fa4e", + "semantic/java.toml": "e7f5b73ffc0fe71d7a9096bdf705ee8cc8e5659e2715cdb1705bd59f6c31dced", + "semantic/javascript.toml": "91c6aa895021aff755c4bf4b4ed4f4c7d8a991d260f1f9d9363be68f2134dfd5", + "semantic/php.toml": "8c7585c1241bd904558389f66decf88c27554107fc7c6e3fa1b0855797e8ea0c", + "semantic/python.toml": "97fd7cfd79ef90a3d51255d951c9f4e1ccabe09d5b6b2574f4ff788556977ab0" +} diff --git a/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/manifest.json b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/manifest.json new file mode 100644 index 0000000..17ab48c --- /dev/null +++ b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/manifest.json @@ -0,0 +1,16 @@ +{ + "acquisition_record_digest": "none", + "analyzer_revision": "dc2832564ae623d0ab558381e7dad4cd17c951b6", + "configuration_digest": "06e9304ee277", + "fact_set_digest": "124871987a06", + "families": [ + "access_control" + ], + "label_isolated": true, + "labels": [], + "population_digest": "c866c0e812c0b54476c7168fdb0e4a20652de0d30485270afecbe51e5c35e424", + "population_role": "admission", + "produced_by": "valid-detection-benchmark", + "reviewed": true, + "spent": false +} diff --git a/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/report.json b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/report.json new file mode 100644 index 0000000..707bac5 --- /dev/null +++ b/benchmarks/measurements/2026-10-31-access-control-admission-unique-binding/report.json @@ -0,0 +1,93 @@ +{ + "coverage": 0.6666666666666666, + "criteria_pre_registered": { + "max_flagged_ordinary_per_100": 10.0, + "max_indeterminate_rate": 0.5, + "min_fix_recall": 0.5, + "min_pair_correctness": 0.5, + "min_precision": 0.25, + "min_verified_location_recall": 0.5 + }, + "decision": "NO-GO", + "failed_gates": [ + "min_fix_recall: 0.0 < 0.5", + "min_pair_correctness: 0.0 < 0.5", + "min_precision: 0.0 < 0.25", + "min_verified_location_recall: not measured (n=0) vs 0.5" + ], + "families": [ + "access_control" + ], + "fix_recall": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "flagged_ordinary_per_100": { + "count": 2, + "n": 240, + "rate": 0.8333333333333334, + "wilson95": [ + 0.2288235363161974, + 2.9870346273441455 + ] + }, + "indeterminate_rate": { + "count": 3, + "n": 264, + "rate": 0.011363636363636364, + "wilson95": [ + 0.003871976570054633, + 0.03287212890123884 + ] + }, + "instrument_failures": 0, + "manifest": { + "acquisition_record_digest": "none", + "analyzer_revision": "dc2832564ae623d0ab558381e7dad4cd17c951b6", + "configuration_digest": "06e9304ee277", + "fact_set_digest": "124871987a06", + "population_digest": "c866c0e812c0b54476c7168fdb0e4a20652de0d30485270afecbe51e5c35e424", + "population_role": "admission" + }, + "pair_correctness": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "pairs": { + "correct": 0, + "n": 12 + }, + "precision": { + "count": 0, + "n": 14, + "rate": 0.0, + "wilson95": [ + 0, + 0.21531701192718142 + ] + }, + "runtime": { + "max_seconds": 7.588272099965252, + "mean_seconds": 2.96003866649941, + "n": 12 + }, + "verified_location_recall": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } +}