docs(benchmarks): 8.2 admission gate - recognition doubled, NO-GO #2, stop condition fired

2026-11-13 17:58:18 +00:00 · 2026-10-30 20:55:38 +01:00
parent 6024152c48
commit 62c8d21667
7 changed files with 226 additions and 0 deletions
@@ -0,0 +1,59 @@
+# Admission with the express entry pack (detection-core 8.2 gate, 2026-10-30)
+
+Second admission run of the access-control arm on the untouched pilot population, with
+the recognition workstream active: the express entry fact pack (inside the frozen fact
+digest) plus the JS handler-extraction fix (bind(this)/arrow/class methods).
+
+## What ran
+
+| Input | Value |
+|---|---|
+| Analyzer revision | a8a28f8 (benchmark plumbing at 6024152; analyzer code identical) |
+| Fact set | 19 artifacts (18 ruleset TOMLs + entries/express.json), digest 1433214dd34e, bundle 124871987a06 |
+| Configuration | (arm=access-control-admission, judgment_mode=strict, entry_pack=express), digest 06e9304ee277 |
+| Population | unchanged pilot pool, freeze c866c0e812c0, guard 0 |
+| Criteria | identical to the 2026-10-30 run (copied verbatim before the run) |
+| Records | 264 fresh proofs in ~/ousast-results/prover/ac-admission-2 (266s, 0 instrument failures) |
+
+## Result: NO-GO — recognition doubled, recall unchanged
+
+| Metric | 2026-10-30 (before) | this run |
+|---|---|---|
+| Decision | NO-GO | **NO-GO** (same four gates failed) |
+| fix_recall | 0/12 | 0/12 |
+| coverage (repos recognized) | 4/12 = 0.333 | **8/12 = 0.667** |
+| dbhub frameworks | 0 | 20 |
+| matrix frameworks | 0 | 4 |
+| ckan frameworks | 0 | 11 |
+| webpack frameworks | 0 | 10 |
+| gitlab-mcp frameworks | 2 | 34 |
+| flagged ordinary / precision | 2/240, 0/14 | 2/240, 0/14 |
+| instrument failures | 0 | 0 |
+
+## Reading
+
+The recognition lever moved exactly what it claimed: framework recognition on the
+express-family repos went from zero to double digits, coverage doubled. It was necessary
+but not sufficient: with entries recognized, the vulnerable sides still produce zero
+judged pairs — the funnel is all-zero because no obligation pairs form at those entries.
+The next bottleneck in the pre-registered ordering (recognition -> resolution -> judgment)
+is **operation facts and call resolution for the express/TS handler shapes**.
+
+## Stop condition fired
+
+This is the **second consecutive admission NO-GO** for this arm against the same
+pre-registered criteria (detection-quality.md rule 10). Per the pre-registered stop
+conditions the arm is **parked**: no third incremental fix round. The next step is a
+maintainer strategy review; the recorded fallback (rule 11) is the narrowed
+one-ecosystem advisory product plus proof-based BLOCK.
+
+Task 8.2's done-when (a pack passing admission gates) is **not met**; the gate ran and
+refused. The task stays open with this run as its outcome.
+
+## Reproduction
+
+```sh
+.venv/bin/python /tmp/opencode/freeze_82gate.py   # verify facts/pool, author manifest+criteria
+.venv/bin/python /tmp/opencode/run_82gate.py      # 264 proofs -> report.json
+.venv/bin/python /tmp/opencode/compare_82.py      # per-repo before/after table
+```
@@ -0,0 +1,21 @@
+{
+  "c/rules.toml": "88ed02da9fef6f170b912f601df12d3040b990d962b28ce4021f3f335442f713",
+  "entries/express.json": "e8e149309abd78f24206c58fde7f73ae2f60364d8b8fee22ecaee8342798d8ce",
+  "families.toml": "5b4c0487c755e8a280e33bbcfb1f497dc40d8703b5961c6f9322a17a3291881a",
+  "frameworks.toml": "cbb9f791e25d73cc215315b2dbfe9654d2ccd5747852e2a0d11ee45ea3012642",
+  "groovy/rules.toml": "ed0ab48e2da2ad9d084a2e5f55ac929e94ea381000d674c160843f79c352396a",
+  "java/rules.toml": "5934e10f595f907f37629d3b05cb56744f70d3c5cb534231b4b3f13a6510c234",
+  "javascript/rules.toml": "075ea7439d65082ecacdf9d9b98397ee278ed93c11ebf552687603d8fb171ce3",
+  "obligations/java.toml": "9125cb707312a72ac766cd3a10e1a09063ac4a2861487ebcb342eadb620e9108",
+  "obligations/javascript.toml": "14670c32a3622b213eb553256a7ffb38c40ffb3398012eb6f02a1e4da04d9a33",
+  "obligations/php.toml": "a7bcbd9bedace740567c5900db91acaa1d0aeb2049dea3101f518acb934827ce",
+  "obligations/python.toml": "f65cb000dbf9e777dce0f2888b0dca39b531b1624dcc3026cc2d55461647b983",
+  "obligations/typescript.toml": "5b9a9bf1bb4c1846a518f724db161b4b5aa533451bc32aaf84ecde851d824bdf",
+  "php/rules.toml": "e839846d760b57c7d65e4fcad6c206e1f1aa852d0ec4bbb9b0b459e761467115",
+  "python/rules.toml": "e323456fd46b5c3e4092a92463695458a620830b6d9b5f05a4ade1e14d78a5c9",
+  "semantic/c.toml": "2d4bbc842596fc17a70e51e528b57af5961ef05515259a57a203faa18018fa4e",
+  "semantic/java.toml": "e7f5b73ffc0fe71d7a9096bdf705ee8cc8e5659e2715cdb1705bd59f6c31dced",
+  "semantic/javascript.toml": "91c6aa895021aff755c4bf4b4ed4f4c7d8a991d260f1f9d9363be68f2134dfd5",
+  "semantic/php.toml": "8c7585c1241bd904558389f66decf88c27554107fc7c6e3fa1b0855797e8ea0c",
+  "semantic/python.toml": "97fd7cfd79ef90a3d51255d951c9f4e1ccabe09d5b6b2574f4ff788556977ab0"
+}
@@ -0,0 +1,8 @@
+{
+  "acquisition_artifact": null,
+  "acquisition_record_digest": "none",
+  "analyzer_revision": "a8a28f81f3c8887ef62b0cf63012de2b9201037f",
+  "configuration_digest": "06e9304ee277",
+  "fact_artifact": "facts-admission.json",
+  "fact_set_digest": "124871987a06"
+}
@@ -0,0 +1,8 @@
+{
+  "max_flagged_ordinary_per_100": 10.0,
+  "max_indeterminate_rate": 0.5,
+  "min_fix_recall": 0.5,
+  "min_pair_correctness": 0.5,
+  "min_precision": 0.25,
+  "min_verified_location_recall": 0.5
+}
@@ -0,0 +1,21 @@
+{
+  "c/rules.toml": "88ed02da9fef6f170b912f601df12d3040b990d962b28ce4021f3f335442f713",
+  "entries/express.json": "e8e149309abd78f24206c58fde7f73ae2f60364d8b8fee22ecaee8342798d8ce",
+  "families.toml": "5b4c0487c755e8a280e33bbcfb1f497dc40d8703b5961c6f9322a17a3291881a",
+  "frameworks.toml": "cbb9f791e25d73cc215315b2dbfe9654d2ccd5747852e2a0d11ee45ea3012642",
+  "groovy/rules.toml": "ed0ab48e2da2ad9d084a2e5f55ac929e94ea381000d674c160843f79c352396a",
+  "java/rules.toml": "5934e10f595f907f37629d3b05cb56744f70d3c5cb534231b4b3f13a6510c234",
+  "javascript/rules.toml": "075ea7439d65082ecacdf9d9b98397ee278ed93c11ebf552687603d8fb171ce3",
+  "obligations/java.toml": "9125cb707312a72ac766cd3a10e1a09063ac4a2861487ebcb342eadb620e9108",
+  "obligations/javascript.toml": "14670c32a3622b213eb553256a7ffb38c40ffb3398012eb6f02a1e4da04d9a33",
+  "obligations/php.toml": "a7bcbd9bedace740567c5900db91acaa1d0aeb2049dea3101f518acb934827ce",
+  "obligations/python.toml": "f65cb000dbf9e777dce0f2888b0dca39b531b1624dcc3026cc2d55461647b983",
+  "obligations/typescript.toml": "5b9a9bf1bb4c1846a518f724db161b4b5aa533451bc32aaf84ecde851d824bdf",
+  "php/rules.toml": "e839846d760b57c7d65e4fcad6c206e1f1aa852d0ec4bbb9b0b459e761467115",
+  "python/rules.toml": "e323456fd46b5c3e4092a92463695458a620830b6d9b5f05a4ade1e14d78a5c9",
+  "semantic/c.toml": "2d4bbc842596fc17a70e51e528b57af5961ef05515259a57a203faa18018fa4e",
+  "semantic/java.toml": "e7f5b73ffc0fe71d7a9096bdf705ee8cc8e5659e2715cdb1705bd59f6c31dced",
+  "semantic/javascript.toml": "91c6aa895021aff755c4bf4b4ed4f4c7d8a991d260f1f9d9363be68f2134dfd5",
+  "semantic/php.toml": "8c7585c1241bd904558389f66decf88c27554107fc7c6e3fa1b0855797e8ea0c",
+  "semantic/python.toml": "97fd7cfd79ef90a3d51255d951c9f4e1ccabe09d5b6b2574f4ff788556977ab0"
+}
@@ -0,0 +1,16 @@
+{
+  "acquisition_record_digest": "none",
+  "analyzer_revision": "a8a28f81f3c8887ef62b0cf63012de2b9201037f",
+  "configuration_digest": "06e9304ee277",
+  "fact_set_digest": "124871987a06",
+  "families": [
+    "access_control"
+  ],
+  "label_isolated": true,
+  "labels": [],
+  "population_digest": "c866c0e812c0b54476c7168fdb0e4a20652de0d30485270afecbe51e5c35e424",
+  "population_role": "admission",
+  "produced_by": "valid-detection-benchmark",
+  "reviewed": true,
+  "spent": false
+}
@@ -0,0 +1,93 @@
+{
+  "coverage": 0.6666666666666666,
+  "criteria_pre_registered": {
+    "max_flagged_ordinary_per_100": 10.0,
+    "max_indeterminate_rate": 0.5,
+    "min_fix_recall": 0.5,
+    "min_pair_correctness": 0.5,
+    "min_precision": 0.25,
+    "min_verified_location_recall": 0.5
+  },
+  "decision": "NO-GO",
+  "failed_gates": [
+    "min_fix_recall: 0.0 < 0.5",
+    "min_pair_correctness: 0.0 < 0.5",
+    "min_precision: 0.0 < 0.25",
+    "min_verified_location_recall: not measured (n=0) vs 0.5"
+  ],
+  "families": [
+    "access_control"
+  ],
+  "fix_recall": {
+    "count": 0,
+    "n": 12,
+    "rate": 0.0,
+    "wilson95": [
+      0,
+      0.2425007574992425
+    ]
+  },
+  "flagged_ordinary_per_100": {
+    "count": 2,
+    "n": 240,
+    "rate": 0.8333333333333334,
+    "wilson95": [
+      0.2288235363161974,
+      2.9870346273441455
+    ]
+  },
+  "indeterminate_rate": {
+    "count": 3,
+    "n": 264,
+    "rate": 0.011363636363636364,
+    "wilson95": [
+      0.003871976570054633,
+      0.03287212890123884
+    ]
+  },
+  "instrument_failures": 0,
+  "manifest": {
+    "acquisition_record_digest": "none",
+    "analyzer_revision": "a8a28f81f3c8887ef62b0cf63012de2b9201037f",
+    "configuration_digest": "06e9304ee277",
+    "fact_set_digest": "124871987a06",
+    "population_digest": "c866c0e812c0b54476c7168fdb0e4a20652de0d30485270afecbe51e5c35e424",
+    "population_role": "admission"
+  },
+  "pair_correctness": {
+    "count": 0,
+    "n": 12,
+    "rate": 0.0,
+    "wilson95": [
+      0,
+      0.2425007574992425
+    ]
+  },
+  "pairs": {
+    "correct": 0,
+    "n": 12
+  },
+  "precision": {
+    "count": 0,
+    "n": 14,
+    "rate": 0.0,
+    "wilson95": [
+      0,
+      0.21531701192718142
+    ]
+  },
+  "runtime": {
+    "max_seconds": 7.2412970759905875,
+    "mean_seconds": 2.9504351955862753,
+    "n": 12
+  },
+  "verified_location_recall": {
+    "count": 0,
+    "n": 0,
+    "rate": null,
+    "wilson95": [
+      null,
+      null
+    ]
+  }
+}