From 62c8d21667be972a5d1b64064208ed2b4f57ff23 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:55:38 +0100 Subject: [PATCH] docs(benchmarks): 8.2 admission gate - recognition doubled, NO-GO #2, stop condition fired --- .../NOTES.md | 59 ++++++++++++ .../bundle/facts-admission.json | 21 +++++ .../bundle/manifest.json | 8 ++ .../criteria.json | 8 ++ .../facts-admission.json | 21 +++++ .../manifest.json | 16 ++++ .../report.json | 93 +++++++++++++++++++ 7 files changed, 226 insertions(+) create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission-express-pack/NOTES.md create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission-express-pack/bundle/facts-admission.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission-express-pack/bundle/manifest.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission-express-pack/criteria.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission-express-pack/facts-admission.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission-express-pack/manifest.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission-express-pack/report.json diff --git a/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/NOTES.md b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/NOTES.md new file mode 100644 index 0000000..02f3ecf --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/NOTES.md @@ -0,0 +1,59 @@ +# Admission with the express entry pack (detection-core 8.2 gate, 2026-10-30) + +Second admission run of the access-control arm on the untouched pilot population, with +the recognition workstream active: the express entry fact pack (inside the frozen fact +digest) plus the JS handler-extraction fix (bind(this)/arrow/class methods). + +## What ran + +| Input | Value | +|---|---| +| Analyzer revision | a8a28f8 (benchmark plumbing at 6024152; analyzer code identical) | +| Fact set | 19 artifacts (18 ruleset TOMLs + entries/express.json), digest 1433214dd34e, bundle 124871987a06 | +| Configuration | (arm=access-control-admission, judgment_mode=strict, entry_pack=express), digest 06e9304ee277 | +| Population | unchanged pilot pool, freeze c866c0e812c0, guard 0 | +| Criteria | identical to the 2026-10-30 run (copied verbatim before the run) | +| Records | 264 fresh proofs in ~/ousast-results/prover/ac-admission-2 (266s, 0 instrument failures) | + +## Result: NO-GO — recognition doubled, recall unchanged + +| Metric | 2026-10-30 (before) | this run | +|---|---|---| +| Decision | NO-GO | **NO-GO** (same four gates failed) | +| fix_recall | 0/12 | 0/12 | +| coverage (repos recognized) | 4/12 = 0.333 | **8/12 = 0.667** | +| dbhub frameworks | 0 | 20 | +| matrix frameworks | 0 | 4 | +| ckan frameworks | 0 | 11 | +| webpack frameworks | 0 | 10 | +| gitlab-mcp frameworks | 2 | 34 | +| flagged ordinary / precision | 2/240, 0/14 | 2/240, 0/14 | +| instrument failures | 0 | 0 | + +## Reading + +The recognition lever moved exactly what it claimed: framework recognition on the +express-family repos went from zero to double digits, coverage doubled. It was necessary +but not sufficient: with entries recognized, the vulnerable sides still produce zero +judged pairs — the funnel is all-zero because no obligation pairs form at those entries. +The next bottleneck in the pre-registered ordering (recognition -> resolution -> judgment) +is **operation facts and call resolution for the express/TS handler shapes**. + +## Stop condition fired + +This is the **second consecutive admission NO-GO** for this arm against the same +pre-registered criteria (detection-quality.md rule 10). Per the pre-registered stop +conditions the arm is **parked**: no third incremental fix round. The next step is a +maintainer strategy review; the recorded fallback (rule 11) is the narrowed +one-ecosystem advisory product plus proof-based BLOCK. + +Task 8.2's done-when (a pack passing admission gates) is **not met**; the gate ran and +refused. The task stays open with this run as its outcome. + +## Reproduction + +```sh +.venv/bin/python /tmp/opencode/freeze_82gate.py # verify facts/pool, author manifest+criteria +.venv/bin/python /tmp/opencode/run_82gate.py # 264 proofs -> report.json +.venv/bin/python /tmp/opencode/compare_82.py # per-repo before/after table +``` diff --git a/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/bundle/facts-admission.json b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/bundle/facts-admission.json new file mode 100644 index 0000000..6e8eded --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/bundle/facts-admission.json @@ -0,0 +1,21 @@ +{ + "c/rules.toml": "88ed02da9fef6f170b912f601df12d3040b990d962b28ce4021f3f335442f713", + "entries/express.json": "e8e149309abd78f24206c58fde7f73ae2f60364d8b8fee22ecaee8342798d8ce", + "families.toml": "5b4c0487c755e8a280e33bbcfb1f497dc40d8703b5961c6f9322a17a3291881a", + "frameworks.toml": "cbb9f791e25d73cc215315b2dbfe9654d2ccd5747852e2a0d11ee45ea3012642", + "groovy/rules.toml": "ed0ab48e2da2ad9d084a2e5f55ac929e94ea381000d674c160843f79c352396a", + "java/rules.toml": "5934e10f595f907f37629d3b05cb56744f70d3c5cb534231b4b3f13a6510c234", + "javascript/rules.toml": "075ea7439d65082ecacdf9d9b98397ee278ed93c11ebf552687603d8fb171ce3", + "obligations/java.toml": "9125cb707312a72ac766cd3a10e1a09063ac4a2861487ebcb342eadb620e9108", + "obligations/javascript.toml": "14670c32a3622b213eb553256a7ffb38c40ffb3398012eb6f02a1e4da04d9a33", + "obligations/php.toml": "a7bcbd9bedace740567c5900db91acaa1d0aeb2049dea3101f518acb934827ce", + "obligations/python.toml": "f65cb000dbf9e777dce0f2888b0dca39b531b1624dcc3026cc2d55461647b983", + "obligations/typescript.toml": "5b9a9bf1bb4c1846a518f724db161b4b5aa533451bc32aaf84ecde851d824bdf", + "php/rules.toml": "e839846d760b57c7d65e4fcad6c206e1f1aa852d0ec4bbb9b0b459e761467115", + "python/rules.toml": "e323456fd46b5c3e4092a92463695458a620830b6d9b5f05a4ade1e14d78a5c9", + "semantic/c.toml": "2d4bbc842596fc17a70e51e528b57af5961ef05515259a57a203faa18018fa4e", + "semantic/java.toml": "e7f5b73ffc0fe71d7a9096bdf705ee8cc8e5659e2715cdb1705bd59f6c31dced", + "semantic/javascript.toml": "91c6aa895021aff755c4bf4b4ed4f4c7d8a991d260f1f9d9363be68f2134dfd5", + "semantic/php.toml": "8c7585c1241bd904558389f66decf88c27554107fc7c6e3fa1b0855797e8ea0c", + "semantic/python.toml": "97fd7cfd79ef90a3d51255d951c9f4e1ccabe09d5b6b2574f4ff788556977ab0" +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/bundle/manifest.json b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/bundle/manifest.json new file mode 100644 index 0000000..86dc42c --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/bundle/manifest.json @@ -0,0 +1,8 @@ +{ + "acquisition_artifact": null, + "acquisition_record_digest": "none", + "analyzer_revision": "a8a28f81f3c8887ef62b0cf63012de2b9201037f", + "configuration_digest": "06e9304ee277", + "fact_artifact": "facts-admission.json", + "fact_set_digest": "124871987a06" +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/criteria.json b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/criteria.json new file mode 100644 index 0000000..6c5bb35 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/criteria.json @@ -0,0 +1,8 @@ +{ + "max_flagged_ordinary_per_100": 10.0, + "max_indeterminate_rate": 0.5, + "min_fix_recall": 0.5, + "min_pair_correctness": 0.5, + "min_precision": 0.25, + "min_verified_location_recall": 0.5 +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/facts-admission.json b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/facts-admission.json new file mode 100644 index 0000000..6e8eded --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/facts-admission.json @@ -0,0 +1,21 @@ +{ + "c/rules.toml": "88ed02da9fef6f170b912f601df12d3040b990d962b28ce4021f3f335442f713", + "entries/express.json": "e8e149309abd78f24206c58fde7f73ae2f60364d8b8fee22ecaee8342798d8ce", + "families.toml": "5b4c0487c755e8a280e33bbcfb1f497dc40d8703b5961c6f9322a17a3291881a", + "frameworks.toml": "cbb9f791e25d73cc215315b2dbfe9654d2ccd5747852e2a0d11ee45ea3012642", + "groovy/rules.toml": "ed0ab48e2da2ad9d084a2e5f55ac929e94ea381000d674c160843f79c352396a", + "java/rules.toml": "5934e10f595f907f37629d3b05cb56744f70d3c5cb534231b4b3f13a6510c234", + "javascript/rules.toml": "075ea7439d65082ecacdf9d9b98397ee278ed93c11ebf552687603d8fb171ce3", + "obligations/java.toml": "9125cb707312a72ac766cd3a10e1a09063ac4a2861487ebcb342eadb620e9108", + "obligations/javascript.toml": "14670c32a3622b213eb553256a7ffb38c40ffb3398012eb6f02a1e4da04d9a33", + "obligations/php.toml": "a7bcbd9bedace740567c5900db91acaa1d0aeb2049dea3101f518acb934827ce", + "obligations/python.toml": "f65cb000dbf9e777dce0f2888b0dca39b531b1624dcc3026cc2d55461647b983", + "obligations/typescript.toml": "5b9a9bf1bb4c1846a518f724db161b4b5aa533451bc32aaf84ecde851d824bdf", + "php/rules.toml": "e839846d760b57c7d65e4fcad6c206e1f1aa852d0ec4bbb9b0b459e761467115", + "python/rules.toml": "e323456fd46b5c3e4092a92463695458a620830b6d9b5f05a4ade1e14d78a5c9", + "semantic/c.toml": "2d4bbc842596fc17a70e51e528b57af5961ef05515259a57a203faa18018fa4e", + "semantic/java.toml": "e7f5b73ffc0fe71d7a9096bdf705ee8cc8e5659e2715cdb1705bd59f6c31dced", + "semantic/javascript.toml": "91c6aa895021aff755c4bf4b4ed4f4c7d8a991d260f1f9d9363be68f2134dfd5", + "semantic/php.toml": "8c7585c1241bd904558389f66decf88c27554107fc7c6e3fa1b0855797e8ea0c", + "semantic/python.toml": "97fd7cfd79ef90a3d51255d951c9f4e1ccabe09d5b6b2574f4ff788556977ab0" +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/manifest.json b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/manifest.json new file mode 100644 index 0000000..8860328 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/manifest.json @@ -0,0 +1,16 @@ +{ + "acquisition_record_digest": "none", + "analyzer_revision": "a8a28f81f3c8887ef62b0cf63012de2b9201037f", + "configuration_digest": "06e9304ee277", + "fact_set_digest": "124871987a06", + "families": [ + "access_control" + ], + "label_isolated": true, + "labels": [], + "population_digest": "c866c0e812c0b54476c7168fdb0e4a20652de0d30485270afecbe51e5c35e424", + "population_role": "admission", + "produced_by": "valid-detection-benchmark", + "reviewed": true, + "spent": false +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/report.json b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/report.json new file mode 100644 index 0000000..8f7d363 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission-express-pack/report.json @@ -0,0 +1,93 @@ +{ + "coverage": 0.6666666666666666, + "criteria_pre_registered": { + "max_flagged_ordinary_per_100": 10.0, + "max_indeterminate_rate": 0.5, + "min_fix_recall": 0.5, + "min_pair_correctness": 0.5, + "min_precision": 0.25, + "min_verified_location_recall": 0.5 + }, + "decision": "NO-GO", + "failed_gates": [ + "min_fix_recall: 0.0 < 0.5", + "min_pair_correctness: 0.0 < 0.5", + "min_precision: 0.0 < 0.25", + "min_verified_location_recall: not measured (n=0) vs 0.5" + ], + "families": [ + "access_control" + ], + "fix_recall": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "flagged_ordinary_per_100": { + "count": 2, + "n": 240, + "rate": 0.8333333333333334, + "wilson95": [ + 0.2288235363161974, + 2.9870346273441455 + ] + }, + "indeterminate_rate": { + "count": 3, + "n": 264, + "rate": 0.011363636363636364, + "wilson95": [ + 0.003871976570054633, + 0.03287212890123884 + ] + }, + "instrument_failures": 0, + "manifest": { + "acquisition_record_digest": "none", + "analyzer_revision": "a8a28f81f3c8887ef62b0cf63012de2b9201037f", + "configuration_digest": "06e9304ee277", + "fact_set_digest": "124871987a06", + "population_digest": "c866c0e812c0b54476c7168fdb0e4a20652de0d30485270afecbe51e5c35e424", + "population_role": "admission" + }, + "pair_correctness": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "pairs": { + "correct": 0, + "n": 12 + }, + "precision": { + "count": 0, + "n": 14, + "rate": 0.0, + "wilson95": [ + 0, + 0.21531701192718142 + ] + }, + "runtime": { + "max_seconds": 7.2412970759905875, + "mean_seconds": 2.9504351955862753, + "n": 12 + }, + "verified_location_recall": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } +}