From 1edc4790afaa1018adbe2af5f8be56783d0f728f Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:25:34 +0100 Subject: [PATCH] feat(access-control-prover): 13.4 admission measurement --- .kiro/specs/access-control-prover/tasks.md | 2 +- .../NOTES.md | 119 ++++++ .../bundle/facts-admission.json | 20 + .../bundle/manifest.json | 8 + .../criteria.json | 8 + .../facts-admission.json | 20 + .../manifest.json | 13 + .../report.json | 90 +++++ benchmarks/valid/admission_run.py | 329 ++++++++++++++++ tests/test_admission_run.py | 352 ++++++++++++++++++ 10 files changed, 960 insertions(+), 1 deletion(-) create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission/NOTES.md create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission/bundle/facts-admission.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission/bundle/manifest.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission/criteria.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission/facts-admission.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission/manifest.json create mode 100644 benchmarks/measurements/2026-10-30-access-control-admission/report.json create mode 100644 benchmarks/valid/admission_run.py create mode 100644 tests/test_admission_run.py diff --git a/.kiro/specs/access-control-prover/tasks.md b/.kiro/specs/access-control-prover/tasks.md index 5c0280e..effc19b 100644 --- a/.kiro/specs/access-control-prover/tasks.md +++ b/.kiro/specs/access-control-prover/tasks.md @@ -322,7 +322,7 @@ its final resolution, dominance, fact-acquisition and measurement semantics with - _Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 6.1, 7.3, 11.8, 12.9_ - _Boundary: Development Measurement_ - _Depends: 12.4, 13.1, 13.2_ -- [ ] 13.4 Run admission from an externally prepared untouched manifest +- [x] 13.4 Run admission from an externally prepared untouched manifest - Consume a reviewed manifest from valid-detection-benchmark after analyzer and facts are frozen. - Verify label isolation, paired vulnerable/fixed behavior, reviewed ordinary labels and every admission gate. - Done when the report returns an evidence-backed GO or NO-GO without modifying facts, strategies or thresholds. diff --git a/benchmarks/measurements/2026-10-30-access-control-admission/NOTES.md b/benchmarks/measurements/2026-10-30-access-control-admission/NOTES.md new file mode 100644 index 0000000..8208fdc --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission/NOTES.md @@ -0,0 +1,119 @@ +# Access-control prover admission measurement (Task 13.4) + +First admission run of the amended access-control prover arm on the untouched pilot +population, consumed through the reviewed `valid-detection-benchmark` manifest. Result: +**NO-GO** — the development score does not generalize to the admission population. + +## What ran + +| Input | Value | +|---|---| +| Admission manifest | `manifest.json` (this directory), consumed and validated by `benchmarks/valid/admission_run.py` | +| Analyzer revision | `d063167b28ed7db2335737fdb0dae0555225ea85` (HEAD at freeze) | +| Fact set | 18 ruleset TOMLs, manifest digest `e7db4031d3d5` (unchanged since the 2026-10-30 four-arm run), bundle artifact digest `3eb5bf699434` | +| Configuration | `(arm=access-control-admission, judgment_mode=strict)`, digest `095b5a4905ad` | +| Population | `benchmarks/valid/pilot/pool-valid.toml`, freeze digest `c866c0e812c0`, held-out guard 0, labels 31 confirmed / 0 unchecked | +| Population contents | 12 repositories, 252 changes (12 vulnerability twins + 240 reviewed ordinary) | +| Isolation | No pilot repository referenced by any fact artifact (verified at freeze); facts trace to pool-train clones and approved documentation only | +| Criteria | Pre-registered in `criteria.json` BEFORE the run: min_fix_recall 0.5, min_pair_correctness 0.5, min_verified_location_recall 0.5, min_precision 0.25, max_flagged_ordinary_per_100 10.0, max_indeterminate_rate 0.5 | + +The freeze script verified facts-on-disk == facts-at-HEAD, re-derived the pool freeze +digest, checked the held-out guard, proved isolation, materialised the bundle and wrote +the manifest and criteria before any prover execution (Req 12.1 ordering). + +## Result + +| Metric | Value | +|---|---| +| Decision | **NO-GO** | +| Failed gates | `min_fix_recall: 0.0 < 0.5`, `min_pair_correctness: 0.0 < 0.5`, `min_precision: 0.0 < 0.25`, `min_verified_location_recall: not measured (n=0)` | +| Pairs | n=12, correct=0 | +| fix_recall | 0/12 (Wilson95 0.0-0.243) | +| flagged_ordinary_per_100 | 2/240 = 0.83 (Wilson95 0.23-2.99) | +| precision | 0/14 | +| indeterminate_rate | 3/264 sides = 1.1% (Wilson95 0.4-3.3) — the three fixed-side locator misses below | +| coverage | 4/12 repositories (0.333) | +| runtime | mean 2.8s, max 7.1s per pair; max ordinary change 7.7s (every change budget-gated at 120s, gate passed) | +| instrument failures | 0 (all 264 records prove nonzero input reads) | + +Fixed-side twin protocol: 9 of 12 labelled functions were located by name at the fix +revision (e.g. dbhub `scanTokenMySQL` shifts from lines 135-143 vulnerable to 158-165 +fixed); the remaining 3 are **locator misses** — the function name is still present at +the fix but the declaration scan cannot name it (md-editor `fenceGroup`, i18next +`extendOptionsWithDefaults`, webpack `chartData`, as const-arrows/getters/import-moved +syntax). Locator misses score `indeterminate` on the fixed side and are never converted +to clean results (design.md:371); a function genuinely absent from the fixed revision +would count as quiet ("removed"). No pair was affected: all 3 fixed records carry empty +evidence and the vulnerable sides were not detected. + +## Reading + +The zero is a **recognition failure, not a judgment failure**. On 9 of 12 repositories +the prover recognized no framework at the vulnerable sites, and every vulnerable-side +judgment funnel came back all-zero: no obligation pair was ever formed, so no guard +evidence and no unguarded evidence existed. Under the pair vocabulary these sites score +as completed-quiet ("discharged" per the documented quiet-or-discharged mapping, Req +12.5); the indeterminate rate of 1.1% counts exactly the three fixed-side locator +misses — nothing on the vulnerable side was incomplete; the engine simply had no +knowledge to apply. The obligation facts were promoted from pool-train +clones (Jenkins/Stapler-shaped Java, plus Spring/Servlet documentation); the admission +population is TypeScript/JavaScript/Python MCP-and-web ecosystems. Four repositories +had any recognition at all (fastapi-users 117, malla 161, jupyterlab-lsp 10 framework +hits, gitlab-mcp 2 on ordinary changes only); none produced a site finding. + +The false-alarm side improved against development (0.83 vs 5.2 flagged/100) and the +budget gate passed everywhere (max 7.1s vs 120s), but with zero recall the arm cannot +enter pre-push admission. This is exactly the generalisation check Requirement 12 +exists to force: the development evidence (recall 34.8% on pool-train) does not +transfer to untouched repositories. + +## Verification + +- Instrument: 264/264 records carry nonzero `hook_bytes_read`; zero zero-byte reads; + 0 instrument failures; `status=prover_covered` everywhere. +- Determinism: 4/4 spot re-proofs (2 vulnerable sites + 2 ordinary) byte-identical on + findings, funnel, status and read counts; the independent reviewer re-proved 4/4 + more, including both flagged ordinary records. +- Unit tests: 24/24 (`tests/test_admission_run.py`). Statics: ruff check, format, + C901 and mypy (6-file anchor set) all exit 0. +- Keyword regression (handover-documented expression): 682 passed + 2 skipped, exit 0. + The full suite still fails `test_no_reserved_repository_is_referenced_by_the_development_tree` + — a pre-existing clean-HEAD failure from committed 2026-10-12 artifacts, documented + at the 2026-10-30 four-arm run and outside this task's boundary. +- Boundary: no fact, strategy or threshold file was modified (`git status` shows only + this measurement directory plus the runner and its tests). + +## Corrections (round-1 review) + +The first review round rejected the task on two points, both remediated: + +1. The fixed-side protocol mapped every locator miss to `fixed_operation="removed"` + and thence to a clean `discharged` judgment — an inflationary path that contradicts + design.md:371. Remediated by the three-state protocol (located / missed / removed, + `classify_fixed_site`), with locator misses scored `indeterminate`. All 12 fixed + sides were re-proved under the new stamps (19s); the three misses are disclosed + above and the report was re-assembled from the resumable records. +2. Ordinary changes are now individually budget-gated (design.md:370), and the + regression count was re-measured under the documented keyword expression. + +No measurement number changed except `indeterminate_rate` (0 -> 3/264, the honest +locator-miss accounting); the decision remains NO-GO on the same four gates. + +## Reproduction + +Records: `~/ousast-results/prover/ac-admission-1/{vulnerable,fixed,ordinary}/.json` +(resumable by input digest; `population_digest` on every record is the authoritative +freeze digest `c866c0e812c0`). + +```sh +.venv/bin/python /tmp/opencode/freeze_134.py # verify + author manifest + criteria +.venv/bin/python /tmp/opencode/run_134.py # prove population, write report.json +.venv/bin/python /tmp/opencode/determinism_134.py # 4/4 reproducibility spot-check +``` + +The run is resumable: re-running `run_134.py` after the round-1 protocol fix reused +the 252 unchanged records and re-proved only the 12 fixed sides. + +Smoke trail: `/tmp/opencode/smoke_134.py` proved one twin pair end-to-end before the +full run (fixed-side function located at shifted lines by name, e.g. dbhub +`scanTokenMySQL` 135-143 vulnerable vs 158-165 fixed). diff --git a/benchmarks/measurements/2026-10-30-access-control-admission/bundle/facts-admission.json b/benchmarks/measurements/2026-10-30-access-control-admission/bundle/facts-admission.json new file mode 100644 index 0000000..3751316 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission/bundle/facts-admission.json @@ -0,0 +1,20 @@ +{ + "c/rules.toml": "88ed02da9fef6f170b912f601df12d3040b990d962b28ce4021f3f335442f713", + "families.toml": "5b4c0487c755e8a280e33bbcfb1f497dc40d8703b5961c6f9322a17a3291881a", + "frameworks.toml": "cbb9f791e25d73cc215315b2dbfe9654d2ccd5747852e2a0d11ee45ea3012642", + "groovy/rules.toml": "ed0ab48e2da2ad9d084a2e5f55ac929e94ea381000d674c160843f79c352396a", + "java/rules.toml": "5934e10f595f907f37629d3b05cb56744f70d3c5cb534231b4b3f13a6510c234", + "javascript/rules.toml": "075ea7439d65082ecacdf9d9b98397ee278ed93c11ebf552687603d8fb171ce3", + "obligations/java.toml": "9125cb707312a72ac766cd3a10e1a09063ac4a2861487ebcb342eadb620e9108", + "obligations/javascript.toml": "14670c32a3622b213eb553256a7ffb38c40ffb3398012eb6f02a1e4da04d9a33", + "obligations/php.toml": "a7bcbd9bedace740567c5900db91acaa1d0aeb2049dea3101f518acb934827ce", + "obligations/python.toml": "f65cb000dbf9e777dce0f2888b0dca39b531b1624dcc3026cc2d55461647b983", + "obligations/typescript.toml": "5b9a9bf1bb4c1846a518f724db161b4b5aa533451bc32aaf84ecde851d824bdf", + "php/rules.toml": "e839846d760b57c7d65e4fcad6c206e1f1aa852d0ec4bbb9b0b459e761467115", + "python/rules.toml": "e323456fd46b5c3e4092a92463695458a620830b6d9b5f05a4ade1e14d78a5c9", + "semantic/c.toml": "2d4bbc842596fc17a70e51e528b57af5961ef05515259a57a203faa18018fa4e", + "semantic/java.toml": "e7f5b73ffc0fe71d7a9096bdf705ee8cc8e5659e2715cdb1705bd59f6c31dced", + "semantic/javascript.toml": "91c6aa895021aff755c4bf4b4ed4f4c7d8a991d260f1f9d9363be68f2134dfd5", + "semantic/php.toml": "8c7585c1241bd904558389f66decf88c27554107fc7c6e3fa1b0855797e8ea0c", + "semantic/python.toml": "97fd7cfd79ef90a3d51255d951c9f4e1ccabe09d5b6b2574f4ff788556977ab0" +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission/bundle/manifest.json b/benchmarks/measurements/2026-10-30-access-control-admission/bundle/manifest.json new file mode 100644 index 0000000..06ef8b5 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission/bundle/manifest.json @@ -0,0 +1,8 @@ +{ + "acquisition_artifact": null, + "acquisition_record_digest": "none", + "analyzer_revision": "d063167b28ed7db2335737fdb0dae0555225ea85", + "configuration_digest": "095b5a4905ad", + "fact_artifact": "facts-admission.json", + "fact_set_digest": "3eb5bf699434" +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission/criteria.json b/benchmarks/measurements/2026-10-30-access-control-admission/criteria.json new file mode 100644 index 0000000..6c5bb35 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission/criteria.json @@ -0,0 +1,8 @@ +{ + "max_flagged_ordinary_per_100": 10.0, + "max_indeterminate_rate": 0.5, + "min_fix_recall": 0.5, + "min_pair_correctness": 0.5, + "min_precision": 0.25, + "min_verified_location_recall": 0.5 +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission/facts-admission.json b/benchmarks/measurements/2026-10-30-access-control-admission/facts-admission.json new file mode 100644 index 0000000..3751316 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission/facts-admission.json @@ -0,0 +1,20 @@ +{ + "c/rules.toml": "88ed02da9fef6f170b912f601df12d3040b990d962b28ce4021f3f335442f713", + "families.toml": "5b4c0487c755e8a280e33bbcfb1f497dc40d8703b5961c6f9322a17a3291881a", + "frameworks.toml": "cbb9f791e25d73cc215315b2dbfe9654d2ccd5747852e2a0d11ee45ea3012642", + "groovy/rules.toml": "ed0ab48e2da2ad9d084a2e5f55ac929e94ea381000d674c160843f79c352396a", + "java/rules.toml": "5934e10f595f907f37629d3b05cb56744f70d3c5cb534231b4b3f13a6510c234", + "javascript/rules.toml": "075ea7439d65082ecacdf9d9b98397ee278ed93c11ebf552687603d8fb171ce3", + "obligations/java.toml": "9125cb707312a72ac766cd3a10e1a09063ac4a2861487ebcb342eadb620e9108", + "obligations/javascript.toml": "14670c32a3622b213eb553256a7ffb38c40ffb3398012eb6f02a1e4da04d9a33", + "obligations/php.toml": "a7bcbd9bedace740567c5900db91acaa1d0aeb2049dea3101f518acb934827ce", + "obligations/python.toml": "f65cb000dbf9e777dce0f2888b0dca39b531b1624dcc3026cc2d55461647b983", + "obligations/typescript.toml": "5b9a9bf1bb4c1846a518f724db161b4b5aa533451bc32aaf84ecde851d824bdf", + "php/rules.toml": "e839846d760b57c7d65e4fcad6c206e1f1aa852d0ec4bbb9b0b459e761467115", + "python/rules.toml": "e323456fd46b5c3e4092a92463695458a620830b6d9b5f05a4ade1e14d78a5c9", + "semantic/c.toml": "2d4bbc842596fc17a70e51e528b57af5961ef05515259a57a203faa18018fa4e", + "semantic/java.toml": "e7f5b73ffc0fe71d7a9096bdf705ee8cc8e5659e2715cdb1705bd59f6c31dced", + "semantic/javascript.toml": "91c6aa895021aff755c4bf4b4ed4f4c7d8a991d260f1f9d9363be68f2134dfd5", + "semantic/php.toml": "8c7585c1241bd904558389f66decf88c27554107fc7c6e3fa1b0855797e8ea0c", + "semantic/python.toml": "97fd7cfd79ef90a3d51255d951c9f4e1ccabe09d5b6b2574f4ff788556977ab0" +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission/manifest.json b/benchmarks/measurements/2026-10-30-access-control-admission/manifest.json new file mode 100644 index 0000000..f52eeda --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission/manifest.json @@ -0,0 +1,13 @@ +{ + "acquisition_record_digest": "none", + "analyzer_revision": "d063167b28ed7db2335737fdb0dae0555225ea85", + "configuration_digest": "095b5a4905ad", + "fact_set_digest": "3eb5bf699434", + "label_isolated": true, + "labels": [], + "population_digest": "c866c0e812c0b54476c7168fdb0e4a20652de0d30485270afecbe51e5c35e424", + "population_role": "admission", + "produced_by": "valid-detection-benchmark", + "reviewed": true, + "spent": false +} diff --git a/benchmarks/measurements/2026-10-30-access-control-admission/report.json b/benchmarks/measurements/2026-10-30-access-control-admission/report.json new file mode 100644 index 0000000..deb36d1 --- /dev/null +++ b/benchmarks/measurements/2026-10-30-access-control-admission/report.json @@ -0,0 +1,90 @@ +{ + "coverage": 0.3333333333333333, + "criteria_pre_registered": { + "max_flagged_ordinary_per_100": 10.0, + "max_indeterminate_rate": 0.5, + "min_fix_recall": 0.5, + "min_pair_correctness": 0.5, + "min_precision": 0.25, + "min_verified_location_recall": 0.5 + }, + "decision": "NO-GO", + "failed_gates": [ + "min_fix_recall: 0.0 < 0.5", + "min_pair_correctness: 0.0 < 0.5", + "min_precision: 0.0 < 0.25", + "min_verified_location_recall: not measured (n=0) vs 0.5" + ], + "fix_recall": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "flagged_ordinary_per_100": { + "count": 2, + "n": 240, + "rate": 0.8333333333333334, + "wilson95": [ + 0.2288235363161974, + 2.9870346273441455 + ] + }, + "indeterminate_rate": { + "count": 3, + "n": 264, + "rate": 0.011363636363636364, + "wilson95": [ + 0.003871976570054633, + 0.03287212890123884 + ] + }, + "instrument_failures": 0, + "manifest": { + "acquisition_record_digest": "none", + "analyzer_revision": "d063167b28ed7db2335737fdb0dae0555225ea85", + "configuration_digest": "095b5a4905ad", + "fact_set_digest": "3eb5bf699434", + "population_digest": "c866c0e812c0b54476c7168fdb0e4a20652de0d30485270afecbe51e5c35e424", + "population_role": "admission" + }, + "pair_correctness": { + "count": 0, + "n": 12, + "rate": 0.0, + "wilson95": [ + 0, + 0.2425007574992425 + ] + }, + "pairs": { + "correct": 0, + "n": 12 + }, + "precision": { + "count": 0, + "n": 14, + "rate": 0.0, + "wilson95": [ + 0, + 0.21531701192718142 + ] + }, + "runtime": { + "max_seconds": 7.112766742007807, + "mean_seconds": 2.8931797610857757, + "n": 12 + }, + "verified_location_recall": { + "count": 0, + "n": 0, + "rate": null, + "wilson95": [ + null, + null + ] + } +} diff --git a/benchmarks/valid/admission_run.py b/benchmarks/valid/admission_run.py new file mode 100644 index 0000000..a45ac5f --- /dev/null +++ b/benchmarks/valid/admission_run.py @@ -0,0 +1,329 @@ +"""Admission measurement runner (Task 13.4, Reqs 12.1-12.9). + +Consumes a reviewed admission manifest for the valid-detection-benchmark +pilot population (authored and attested by the benchmark workflow, never by +this module), proves both sides of every vulnerable/fixed twin plus every +reviewed ordinary change on the strict judgment lane with nonzero-read +instrumentation, and returns the paired admission report. Judgment mapping +onto the three-way vocabulary: ``unguarded`` is a finding at the labelled +site; ``indeterminate`` is incomplete site evidence (an incomplete trace, a +degraded site file, or a fixed-side locator miss); every other completed +site is ``discharged`` -- Req 12.5's "quiet or discharged" behaviour. The +runner never modifies facts, strategies or thresholds, and never produces, +mutates or re-labels the manifest it consumes. +""" + +from __future__ import annotations + +import hashlib +import json +import tomllib +from pathlib import Path +from typing import Any + +from benchmarks.unseen.replay import canonical +from benchmarks.valid.admission_scoring import AdmissionCase, OrdinaryChange, admission_report +from benchmarks.valid.arm_regression import BundleDigests, _input_digest, _prove_one +from benchmarks.valid.prover_admission import AdmissionManifest, validate_admission_manifest +from openultrasast.preprocess import detect_language +from openultrasast.source import canonical_language, ranges + +__all__ = [ + "admission_ordinary", + "admission_pair_cases", + "assemble_admission_report", + "coverage_share", + "fixed_side_change", + "load_admission_population", + "locate_function", + "run_admission", + "site_judgment", +] + +ARM_NAME = "access-control-admission" + + +def _site_traces(record: dict[str, Any], path: str, start: int, end: int) -> list[dict[str, Any]]: + """Traces whose operation site falls inside the labelled range.""" + matched = [] + for trace in record.get("traces", ()): + fact_id = str(trace.get("fact_id", "")) + marker = f":{path}:" + index = fact_id.find(marker) + if index < 0: + continue + line_text = fact_id[index + len(marker) :].split(":", 1)[0] + if line_text.isdigit() and start <= int(line_text) <= end: + matched.append(trace) + return matched + + +def site_judgment(record: dict[str, Any], *, path: str, start: int, end: int) -> str: + """Three-way judgment of one labelled site (Req 12.5 pair semantics).""" + findings = record.get("findings", ()) + if any(finding.get("path") == path and start <= int(finding.get("line", 0)) <= end for finding in findings): + return "unguarded" + judgments = [trace["abstract_state"]["path_judgment"] for trace in _site_traces(record, path, start, end)] + if any(judgment.get("judgment") == "unguarded" for judgment in judgments): + return "unguarded" + incomplete = any( + judgment.get("judgment") == "indeterminate" or not judgment.get("completeness", {}).get("is_complete", True) + for judgment in judgments + ) + if incomplete: + return "indeterminate" + if any(degradation.get("path") == path for degradation in record.get("degradations", ())): + return "indeterminate" + return "discharged" + + +def locate_function(functions: Any, *, name: str) -> tuple[int, int] | None: + """Inclusive line range of ``name`` among function ranges, or None.""" + for function in functions: + label = function[0] if isinstance(function, tuple) else function.name + if label == name: + start = function[1] if isinstance(function, tuple) else function.start + end = function[2] if isinstance(function, tuple) else function.end + return int(start), int(end) + return None + + +def fixed_side_change(change: dict[str, Any], *, fixed_lines: tuple[int, int] | None, fixed_operation: str) -> dict[str, Any]: + """The twin proof target: the fix commit checked out with its own site lines. + + ``fixed_operation`` is "located" with lines, or one of "missed" (the + function name is present at the fix but the locator could not name it -- + const-arrows, getters, moved imports) and "removed" (the name is absent + from the fixed revision entirely). Only "removed" counts as quiet. + """ + site = str(change.get("path") or change["files"][0]["path"]) + fixed: dict[str, Any] = { + **change, + "base": str(change["head"]), + "head": str(change["base"]), + "fixed_operation": fixed_operation, + "files": [{"path": site, "head_lines": [list(fixed_lines)]}] if fixed_lines else [], + } + if fixed_lines: + fixed["fixed_lines"] = list(fixed_lines) + return fixed + + +def classify_fixed_site(lines: tuple[int, int] | None, *, name_in_text: bool) -> str: + """Located, or missed (name present, locator failed) vs removed (name absent).""" + if lines is not None: + return "located" + return "missed" if name_in_text else "removed" + + +def _fixed_judgment(record: dict[str, Any]) -> str: + if record.get("fixed_operation") == "removed": + return "discharged" + if "fixed_lines" not in record: + return "indeterminate" + start, end = (int(value) for value in record["fixed_lines"]) + return site_judgment(record, path=str(record["path"]), start=start, end=end) + + +def admission_pair_cases(vulnerable: tuple[dict[str, Any], ...], fixed: tuple[dict[str, Any], ...]) -> tuple[AdmissionCase, ...]: + """Build one scored twin pair per vulnerable case (Req 12.5).""" + fixed_by_id = {str(record["id"]): record for record in fixed} + cases = [] + for record in vulnerable: + counterpart = fixed_by_id[str(record["id"])] + start, end = (int(value) for value in record["function_lines"]) + path = str(record["path"]) + verified = any( + finding.get("path") == path and start <= int(finding.get("line", 0)) <= end and finding.get("family") == record.get("family") + for finding in record.get("findings", ()) + ) + cases.append( + AdmissionCase( + case_id=str(record["id"]), + vulnerable_judgment=site_judgment(record, path=path, start=start, end=end), + fixed_judgment=_fixed_judgment(counterpart), + location_verified=bool(verified), + seconds=float(record.get("seconds", 0.0)) + float(counterpart.get("seconds", 0.0)), + ) + ) + return tuple(cases) + + +def admission_ordinary(records: tuple[dict[str, Any], ...]) -> tuple[OrdinaryChange, ...]: + """Reviewed ordinary changes form the false-alarm denominator (Req 12.4).""" + changes = [] + for record in records: + if record.get("findings"): + judgment = "unguarded" + elif int(record.get("stage_funnel", {}).get("judgment", {}).get("indeterminate", 0)) > 0: + judgment = "indeterminate" + else: + judgment = "discharged" + changes.append(OrdinaryChange(change_id=str(record["id"]), judgment=judgment, mislabeled_defect=False)) + return tuple(changes) + + +def coverage_share(records: tuple[dict[str, Any], ...]) -> float: + """Share of population repositories with any recognised framework.""" + per_repository: dict[str, int] = {} + for record in records: + recognised = record.get("stage_funnel", {}).get("entry_recognition", {}).get("frameworks_recognised", {}) + total = sum(recognised.values()) if isinstance(recognised, dict) else int(recognised) + per_repository[str(record["repo"])] = per_repository.get(str(record["repo"]), 0) + int(total) + if not per_repository: + return 0.0 + covered = sum(1 for total in per_repository.values() if total > 0) + return covered / len(per_repository) + + +def load_admission_population( + payload: dict[str, object], pool_path: Path, freeze_path: Path +) -> tuple[tuple[dict[str, Any], ...], tuple[dict[str, Any], ...]]: + """Validate the manifest and the reviewed freeze before opening labels (Req 12.1).""" + manifest = validate_admission_manifest(payload) + pool = tomllib.loads(Path(pool_path).read_text(encoding="utf-8")) + freeze = json.loads(Path(freeze_path).read_text(encoding="utf-8")) + digest = hashlib.sha256(canonical(pool)).hexdigest() + if digest != freeze.get("freeze_digest"): + raise ValueError(f"population: pool digest {digest[:12]} does not match the reviewed freeze") + if freeze.get("guard_result") != 0: + raise ValueError("guard: held-out guard failed for this population") + if manifest.population_digest != digest: + raise ValueError("population: manifest names a different population than the reviewed freeze") + changes = tuple( + {**change, "id": _change_identity(repository, change), "repo": str(repository["id"])} + for repository in pool.get("repository", []) + for change in repository.get("changes", []) + ) + vulnerable = tuple(change for change in changes if change.get("kind") == "vulnerability") + ordinary = tuple(change for change in changes if change.get("kind") == "ordinary") + return vulnerable, ordinary + + +def _change_identity(repository: dict[str, Any], change: dict[str, Any]) -> str: + """The pool's canonical change identity, as ``replay.load_slice`` derives it.""" + payload = [repository["id"], change["kind"], change["base"], change["head"]] + return hashlib.sha256(canonical(payload)).hexdigest() + + +def assemble_admission_report( + manifest: AdmissionManifest, + vulnerable: tuple[dict[str, Any], ...], + fixed: tuple[dict[str, Any], ...], + ordinary: tuple[dict[str, Any], ...], + *, + criteria: dict[str, float] | None, + budget_seconds: float = 120.0, +) -> dict[str, Any]: + """Bridge measured records to the paired admission report (Reqs 12.7, 12.8). + + Every change is budget-gated: pairs by their combined seconds, ordinary + changes individually (design.md:370 "fails any change over 120 seconds"). + """ + failures = sum(1 for record in (*vulnerable, *fixed, *ordinary) if record.get("status") == "instrument_failure") + report = admission_report( + manifest, + admission_pair_cases(vulnerable, fixed), + admission_ordinary(ordinary), + coverage=coverage_share((*vulnerable, *fixed, *ordinary)), + instrument_failures=failures, + budget_seconds=budget_seconds, + criteria=criteria, + ) + overruns = [ + f"budget:{record['id']}:{float(record.get('seconds', 0.0)):g}s>{budget_seconds:g}s" + for record in ordinary + if float(record.get("seconds", 0.0)) > budget_seconds + ] + if overruns: + report["failed_gates"] = [*report["failed_gates"], *overruns] + report["decision"] = "NO-GO" + return report + + +def _resume_or_prove( + bundle: BundleDigests, + population: str, + change: dict[str, Any], + repo_path: Path, + side_dir: Path, +) -> dict[str, Any]: + from benchmarks.valid.prover_arm import _write_json + + side_dir.mkdir(parents=True, exist_ok=True) + path = side_dir / f"{change['id']}.json" + if path.is_file(): + previous: dict[str, Any] = json.loads(path.read_text(encoding="utf-8")) + if previous.get("input_digest") == _input_digest(change, ARM_NAME): + return previous + raise ValueError(f"existing record for {change['id']} does not match this arm's inputs") + record = _prove_one(bundle, ARM_NAME, population, change, repo_path) + _write_json(path, record) + return record + + +def run_admission( + *, + payload: dict[str, object], + pool_path: Path, + freeze_path: Path, + repos: dict[str, str], + bundle: BundleDigests, + out_dir: Path, + criteria: dict[str, float] | None, + budget_seconds: float = 120.0, +) -> dict[str, Any]: + """Consume the reviewed manifest, prove the population, return GO/NO-GO.""" + vulnerable, ordinary = load_admission_population(payload, pool_path, freeze_path) + manifest = validate_admission_manifest(payload) + out_dir.mkdir(parents=True, exist_ok=True) + vulnerable_records = tuple( + _resume_or_prove(bundle, manifest.population_digest, change, Path(repos[change["repo"]]).expanduser(), out_dir / "vulnerable") + for change in vulnerable + ) + fixed_records = tuple( + _resume_or_prove( + bundle, + manifest.population_digest, + _fixed_side_change(Path(repos[change["repo"]]).expanduser(), change), + Path(repos[change["repo"]]).expanduser(), + out_dir / "fixed", + ) + for change in vulnerable + ) + ordinary_records = tuple( + _resume_or_prove(bundle, manifest.population_digest, change, Path(repos[change["repo"]]).expanduser(), out_dir / "ordinary") + for change in ordinary + ) + return assemble_admission_report( + manifest, vulnerable_records, fixed_records, ordinary_records, criteria=criteria, budget_seconds=budget_seconds + ) + + +def _fixed_side_change(repo_path: Path, change: dict[str, Any]) -> dict[str, Any]: + lines, operation = _fixed_site(repo_path, change) + return fixed_side_change(change, fixed_lines=lines, fixed_operation=operation) + + +def _fixed_site(repo_path: Path, change: dict[str, Any]) -> tuple[tuple[int, int] | None, str]: + """(lines, operation) for the labelled function at the fixed revision. + + A locator miss (the name is present but ``ranges`` could not name it) is + "missed" and can never become a clean judgment (design.md:371). + """ + from benchmarks.valid.catalog import local_git + + git = local_git(repo_path) + text = git.runner( + ["git", "-C", str(repo_path), "show", f"{change['base']}:{change['path']}"], + capture_output=True, + text=True, + check=True, + ).stdout + name = str(change.get("function")) + try: + functions = ranges(str(change["path"]), text, canonical_language(detect_language(repo_path / change["path"]))) + except ValueError: + functions = () + lines = locate_function(functions, name=name) + return lines, classify_fixed_site(lines, name_in_text=name in text) diff --git a/tests/test_admission_run.py b/tests/test_admission_run.py new file mode 100644 index 0000000..93e4151 --- /dev/null +++ b/tests/test_admission_run.py @@ -0,0 +1,352 @@ +"""Task 13.4 admission measurement unit tests (Reqs 12.1-12.9). + +The admission runner consumes a reviewed upstream manifest for the untouched +pilot population, proves both sides of every vulnerable/fixed twin and every +reviewed ordinary change, maps per-site prover evidence onto the three-way +judgment vocabulary and returns the paired admission report. +""" + +import hashlib +import json +import sys +import tempfile +import tomllib +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from benchmarks.unseen.replay import canonical # noqa: E402 +from benchmarks.valid.admission_run import ( # noqa: E402 + admission_ordinary, + admission_pair_cases, + assemble_admission_report, + classify_fixed_site, + coverage_share, + fixed_side_change, + load_admission_population, + locate_function, + site_judgment, +) +from benchmarks.valid.admission_scoring import AdmissionScoringError # noqa: E402 +from benchmarks.valid.prover_admission import AdmissionManifest, AdmissionManifestError # noqa: E402 + +SITE = ("src/app.py", 30, 40) + + +def _trace(path: str, line: int, judgment: str, *, complete: bool = True) -> dict[str, object]: + return { + "fact_id": f"obligation:protected_read:{path}:{line}:identity_constraint", + "abstract_state": { + "path_judgment": { + "judgment": judgment, + "completeness": {"is_complete": complete, "reasons": [] if complete else ["deadline"]}, + } + }, + } + + +def _finding(path: str, line: int, family: str = "access_control") -> dict[str, object]: + return {"path": path, "line": line, "family": family, "id": f"obligation:{path}:{line}"} + + +def _record(record_id: str, **over: object) -> dict[str, Any]: + record = { + "id": record_id, + "repo": "example/project", + "kind": "vulnerability", + "path": "src/app.py", + "function_lines": [30, 40], + "findings": [], + "traces": [], + "degradations": [], + "stage_funnel": {"judgment": {"unguarded": 0, "discharged": 0, "indeterminate": 0}}, + "instrument": {"hook_bytes_read": 10, "changed_bytes": {"src/app.py": 10}, "file_bytes": {"src/app.py": 10}}, + "status": "prover_covered", + "seconds": 1.0, + } + record.update(over) + return record + + +def test_site_judgment_unguarded_when_finding_in_range() -> None: + record = _record("v1", findings=[_finding("src/app.py", 35)]) + assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "unguarded" + + +def test_site_judgment_ignores_finding_outside_site() -> None: + record = _record("v1", findings=[_finding("src/other.py", 35), _finding("src/app.py", 200)]) + assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "discharged" + + +def test_site_judgment_indeterminate_on_incomplete_trace() -> None: + record = _record("v1", traces=[_trace("src/app.py", 33, "indeterminate", complete=False)]) + assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "indeterminate" + + +def test_site_judgment_discharged_on_completed_trace() -> None: + record = _record("v1", traces=[_trace("src/app.py", 33, "discharged", complete=True)]) + assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "discharged" + + +def test_site_judgment_quiet_without_traces_is_discharged() -> None: + assert site_judgment(_record("v1"), path=SITE[0], start=SITE[1], end=SITE[2]) == "discharged" + + +def test_site_judgment_indeterminate_on_degraded_site_file() -> None: + record = _record("v1", degradations=[{"path": "src/app.py", "reason": "parse_failed"}]) + assert site_judgment(record, path=SITE[0], start=SITE[1], end=SITE[2]) == "indeterminate" + + +def test_locate_function_by_name() -> None: + functions = [("helper", 1, 10, "function"), ("target", 30, 40, "function")] + assert locate_function(functions, name="target") == (30, 40) + assert locate_function(functions, name="absent") is None + + +def test_fixed_side_change_swaps_revisions() -> None: + change = { + "id": "case-1", + "base": "fixcommit", + "head": "vulnparent", + "files": [{"path": "src/app.py", "head_lines": [[30, 40]]}], + "kind": "vulnerability", + } + fixed = fixed_side_change(change, fixed_lines=(31, 42), fixed_operation="located") + assert fixed["head"] == "fixcommit" + assert fixed["base"] == "vulnparent" + assert fixed["files"] == [{"path": "src/app.py", "head_lines": [[31, 42]]}] + assert fixed["kind"] == "vulnerability" + assert fixed["fixed_lines"] == [31, 42] + + +def test_classify_fixed_site_missed_vs_removed() -> None: + # const-arrow / getter / moved-import: name present, locator failed. + assert classify_fixed_site(None, name_in_text=True) == "missed" + assert classify_fixed_site(None, name_in_text=False) == "removed" + assert classify_fixed_site((31, 42), name_in_text=True) == "located" + + +def test_pair_case_correct_uses_site_judgments_and_sums_seconds() -> None: + vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)], seconds=2.0, family="access_control") + fixed = _record( + "case-1", + kind="vulnerability", + fixed_lines=(31, 42), + traces=[_trace("src/app.py", 33, "discharged")], + seconds=3.0, + ) + cases = admission_pair_cases((vulnerable,), (fixed,)) + assert len(cases) == 1 + assert cases[0].case_id == "case-1" + assert cases[0].vulnerable_judgment == "unguarded" + assert cases[0].fixed_judgment == "discharged" + assert cases[0].location_verified is True + assert cases[0].seconds == 5.0 + + +def test_pair_case_wrong_family_is_not_location_verified() -> None: + vulnerable = _record("case-1", findings=[_finding("src/app.py", 35, family="output_encoding")]) + fixed = _record("case-1", fixed_lines=(31, 42)) + cases = admission_pair_cases((vulnerable,), (fixed,)) + assert cases[0].vulnerable_judgment == "unguarded" + assert cases[0].location_verified is False + + +def test_removed_fixed_operation_is_quiet() -> None: + vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)]) + fixed = _record("case-1", fixed_operation="removed") + cases = admission_pair_cases((vulnerable,), (fixed,)) + assert cases[0].fixed_judgment == "discharged" + + +def test_locator_miss_fixed_operation_is_never_clean() -> None: + vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)]) + fixed = _record("case-1", fixed_operation="missed") + cases = admission_pair_cases((vulnerable,), (fixed,)) + assert cases[0].fixed_judgment == "indeterminate" + + +def test_unreadable_fixed_operation_is_indeterminate() -> None: + vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)]) + fixed = _record("case-1") + cases = admission_pair_cases((vulnerable,), (fixed,)) + assert cases[0].fixed_judgment == "indeterminate" + + +def test_ordinary_change_flagged_on_any_finding() -> None: + record = _record("ord-1", kind="ordinary", findings=[_finding("src/any.py", 5)]) + changes = admission_ordinary((record,)) + assert changes[0].judgment == "unguarded" + assert changes[0].mislabeled_defect is False + + +def test_ordinary_change_quiet_and_indeterminate() -> None: + quiet = admission_ordinary((_record("ord-1", kind="ordinary"),))[0] + assert quiet.judgment == "discharged" + funnel = {"judgment": {"unguarded": 0, "discharged": 0, "indeterminate": 2}} + undecided = admission_ordinary((_record("ord-2", kind="ordinary", stage_funnel=funnel),))[0] + assert undecided.judgment == "indeterminate" + + +def test_coverage_share_over_repositories() -> None: + covered = _record("a", repo="repo-a", stage_funnel={"entry_recognition": {"frameworks_recognised": {"jenkins": 1}}}) + bare = _record("b", repo="repo-b", stage_funnel={"entry_recognition": {"frameworks_recognised": {}}}) + other = _record("a", repo="repo-a", stage_funnel={"entry_recognition": {"frameworks_recognised": {"jenkins": 2}}}) + assert coverage_share((covered, bare, other)) == 0.5 + + +def _manifest_payload(**over: object) -> dict[str, object]: + payload = { + "produced_by": "valid-detection-benchmark", + "reviewed": True, + "label_isolated": True, + "spent": False, + "population_role": "admission", + "analyzer_revision": "d063167", + "fact_set_digest": "3eb5bf699434", + "acquisition_record_digest": "none", + "configuration_digest": "be6d0f688598", + "population_digest": "0" * 64, + "labels": [], + } + payload.update(over) + return payload + + +POOL_TOML = ( + 'status = "frozen"\n' + 'pool = "valid"\n' + "seed = 0\n" + "\n" + "[[repository]]\n" + 'id = "example/project"\n' + 'repository = "example/project"\n' + 'url = "https://github.com/example/project.git"\n' + "slice = 1\n" + "changes = [\n" + ' {base = "fixcommit", head = "vulnparent", case_id = "example-project-vuln-1", ' + 'id = "example-project-vuln-1", repo = "example/project", kind = "vulnerability", ' + 'path = "src/app.py", function = "handler", function_lines = [30, 40], ' + 'files = [{path = "src/app.py", head_lines = [[30, 40]]}], family = "access_control"},\n' + ' {base = "parent1", head = "ordinary1", case_id = "example-project-ordinary-1", ' + 'id = "example-project-ordinary-1", repo = "example/project", kind = "ordinary", ' + 'files = [{path = "src/lib.py", head_lines = [[1, 5]]}]},\n' + "]\n" +) + + +def _population_files(root: Path, *, freeze_digest: str | None = None, guard_result: int = 0) -> tuple[Path, Path, str]: + root.mkdir(parents=True, exist_ok=True) + pool_path = root / "pool.toml" + pool_path.write_text(POOL_TOML, encoding="utf-8") + pool = tomllib.loads(pool_path.read_text(encoding="utf-8")) + digest = freeze_digest if freeze_digest is not None else hashlib.sha256(canonical(pool)).hexdigest() + freeze_path = root / "freeze.json" + freeze_path.write_text(json.dumps({"freeze_digest": digest, "guard_result": guard_result}), encoding="utf-8") + return pool_path, freeze_path, digest + + +def test_load_population_verifies_freeze_digest_and_guard() -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + pool_path, freeze_path, digest = _population_files(root) + payload = _manifest_payload(population_digest=digest) + vulnerable, ordinary = load_admission_population(payload, pool_path, freeze_path) + assert [change["kind"] for change in (*vulnerable, *ordinary)] == ["vulnerability", "ordinary"] + assert all(len(change["id"]) == 64 for change in (*vulnerable, *ordinary)) + tampered = _manifest_payload(population_digest="f" * 64) + try: + load_admission_population(tampered, pool_path, freeze_path) + raise AssertionError("population digest mismatch must be refused") + except ValueError as error: + assert "population" in str(error) + guarded = _population_files(root / "g", guard_result=1) + payload_guarded = _manifest_payload(population_digest=guarded[2]) + try: + load_admission_population(payload_guarded, guarded[0], guarded[1]) + raise AssertionError("held-out guard failure must be refused") + except ValueError as error: + assert "guard" in str(error) + + +def test_load_population_validates_manifest_before_labels() -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + pool_path, freeze_path, _ = _population_files(root) + spent = _manifest_payload(spent=True) + try: + load_admission_population(spent, pool_path, freeze_path) + raise AssertionError("spent population must be refused") + except AdmissionManifestError: + pass + + +def _admission_manifest(digest: str = "0" * 64) -> AdmissionManifest: + from benchmarks.valid.prover_admission import validate_admission_manifest + + return validate_admission_manifest(_manifest_payload(population_digest=digest)) + + +def test_assemble_report_returns_go_without_failed_gates() -> None: + vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)], family="access_control") + fixed = _record("case-1", fixed_lines=(31, 42), traces=[_trace("src/app.py", 33, "discharged")]) + ordinary = _record("ord-1", kind="ordinary") + manifest = _admission_manifest() + report = assemble_admission_report( + manifest, + (vulnerable,), + (fixed,), + (ordinary,), + criteria={"min_fix_recall": 1.0, "max_flagged_ordinary_per_100": 0.0}, + ) + assert report["decision"] == "GO" + assert report["fix_recall"]["count"] == 1 + assert report["pairs"] == {"n": 1, "correct": 1} + assert report["flagged_ordinary_per_100"]["count"] == 0 + + +def test_assemble_report_no_go_on_ordinary_overrun() -> None: + vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)]) + fixed = _record("case-1", fixed_lines=(31, 42), traces=[_trace("src/app.py", 33, "discharged")]) + slow = _record("ord-1", kind="ordinary", seconds=300.0) + report = assemble_admission_report(_admission_manifest(), (vulnerable,), (fixed,), (slow,), criteria=None, budget_seconds=120.0) + assert report["decision"] == "NO-GO" + assert any(gate.startswith("budget:ord-1:") for gate in report["failed_gates"]) + + +def test_assemble_report_no_go_on_budget_and_criteria() -> None: + vulnerable = _record("case-1", seconds=200.0) + fixed = _record("case-1", fixed_lines=(31, 42), seconds=1.0) + ordinary = _record("ord-1", kind="ordinary", findings=[_finding("src/x.py", 3)]) + manifest = _admission_manifest() + report = assemble_admission_report( + manifest, + (vulnerable,), + (fixed,), + (ordinary,), + criteria={"min_fix_recall": 0.9, "max_flagged_ordinary_per_100": 0.0}, + ) + assert report["decision"] == "NO-GO" + assert any(gate.startswith("budget:case-1:") for gate in report["failed_gates"]) + assert any("min_fix_recall" in gate for gate in report["failed_gates"]) + assert any("max_flagged_ordinary_per_100" in gate for gate in report["failed_gates"]) + + +def test_assemble_report_counts_instrument_failures() -> None: + vulnerable = _record("case-1", status="instrument_failure") + fixed = _record("case-1", fixed_lines=(31, 42), status="instrument_failure") + manifest = _admission_manifest() + report = assemble_admission_report(manifest, (vulnerable,), (fixed,), (), criteria=None) + assert report["instrument_failures"] == 2 + assert report["decision"] == "NO-GO" + + +def test_assemble_report_refuses_development_role() -> None: + manifest = _admission_manifest() + object.__setattr__(manifest, "population_role", "development") + try: + assemble_admission_report(manifest, (), (), (), criteria=None) + raise AssertionError("development population must be refused for admission") + except AdmissionScoringError: + pass