feat(pre-push-safety-net): preserve uncertainty in benchmark scoring

benchmarks/ranking/baselines/2026-09-12-vampi-position-v2.json

@@ -0,0 +1,132 @@
+[
+  {
+    "schema_version": 2,
+    "measurement": "positional_coverage_not_detection",
+    "ranking_mode": "static",
+    "profile": {
+      "checkout": {
+        "commit": "f16052dce83f05847133ec98f01c5193a41de7d8",
+        "clean": true,
+        "status_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+      },
+      "version": "position-v2",
+      "requested_ranking_mode": "static",
+      "known_targets": [
+        {
+          "id": "VAmPI-SQLI",
+          "family": "injection",
+          "file": "models/user_model.py",
+          "function": "get_user",
+          "line": 73,
+          "in_scope": true
+        },
+        {
+          "id": "VAmPI-BOLA",
+          "family": "access_control",
+          "file": "api_views/users.py",
+          "function": "update_password",
+          "line": 187,
+          "in_scope": true
+        },
+        {
+          "id": "VAmPI-BOLA-BOOKS",
+          "family": "access_control",
+          "file": "api_views/books.py",
+          "function": "get_by_title",
+          "line": 51,
+          "in_scope": true
+        }
+      ],
+      "input_witnesses": [
+        {
+          "path": "api_views/books.py",
+          "bytes_read": 3043,
+          "sha256": "78e2eb77d655573f03b7a4d0bca73dc27383b0c5ec7882a5aa8608caa52828e8"
+        },
+        {
+          "path": "api_views/users.py",
+          "bytes_read": 9257,
+          "sha256": "46006eb7207ebbae163f5320b13e3bf0ec464cef201a51fb2cd43b5cfdb4116c"
+        },
+        {
+          "path": "models/user_model.py",
+          "bytes_read": 3642,
+          "sha256": "4ce47b9850812e47140b2d2f2e20c08ae4b5338d168ce72f23e3127986ef6e1b"
+        }
+      ],
+      "implementation": {
+        "benchmarks/ranking/position.py": "c362e58843e669b68bb032574beff2342bd9597e0b451b38c727c1af643450e2",
+        "src/openultrasast/model/regions.py": "0ea1a7b153fad403848aa03f54ae45ea0dcf9c3d8b0f710238362de05139c84b",
+        "src/openultrasast/model/scan.py": "e11423755be4b72cfb6b5fd86b3d2fef68c0cc33aab997210f51430570256e93",
+        "src/openultrasast/model/evidence.py": "cda3619170a77bb8a4b7bb7fecb261e990699fe671714ccc830505a81029788d",
+        "src/openultrasast/model/layout.py": "dc9dc338dff5f1d4389d59d84ba4ddd249949470f9c2303df5ee70ad5d257f2d"
+      },
+      "detection": "not_measured",
+      "sha256": "273dd99adcd94ec93df5e4c30018c145092141ba2c4e28a409d8bb5f8e6939a2"
+    },
+    "detection_recall": null,
+    "supported_population": 3,
+    "unmatched_supported": 1,
+    "input_witnesses": [
+      {
+        "path": "api_views/books.py",
+        "bytes_read": 3043,
+        "sha256": "78e2eb77d655573f03b7a4d0bca73dc27383b0c5ec7882a5aa8608caa52828e8"
+      },
+      {
+        "path": "api_views/users.py",
+        "bytes_read": 9257,
+        "sha256": "46006eb7207ebbae163f5320b13e3bf0ec464cef201a51fb2cd43b5cfdb4116c"
+      },
+      {
+        "path": "models/user_model.py",
+        "bytes_read": 3642,
+        "sha256": "4ce47b9850812e47140b2d2f2e20c08ae4b5338d168ce72f23e3127986ef6e1b"
+      }
+    ],
+    "input_status": "readable",
+    "total_seconds": 0.793,
+    "repo": "vampi",
+    "regions": 25,
+    "distinct_ranks": 5,
+    "largest_tier_share": 0.36,
+    "largest_tier_rank": 1.0,
+    "budget_at_recall": null,
+    "budget_at_full_target_position_coverage": null,
+    "known": [
+      {
+        "id": "VAmPI-SQLI",
+        "family": "injection",
+        "site": "models/user_model.py:73:get_user",
+        "position": null,
+        "rank": null,
+        "in_scope": true,
+        "reached_transitively": false,
+        "detection": "not_measured",
+        "position_status": "unresolved_target"
+      },
+      {
+        "id": "VAmPI-BOLA",
+        "family": "access_control",
+        "site": "api_views/users.py:187:update_password",
+        "position": 15,
+        "rank": 0.8,
+        "in_scope": true,
+        "reached_transitively": false,
+        "detection": "not_measured",
+        "position_status": "matched"
+      },
+      {
+        "id": "VAmPI-BOLA-BOOKS",
+        "family": "access_control",
+        "site": "api_views/books.py:51:get_by_title",
+        "position": 10,
+        "rank": 0.8,
+        "in_scope": true,
+        "reached_transitively": false,
+        "detection": "not_measured",
+        "position_status": "matched"
+      }
+    ]
+  }
+]

benchmarks/ranking/baselines/README.md

@@ -0,0 +1,20 @@
+# Historical ranking measurements
+
+The eight JSON artifacts dated 2026-09-10 and 2026-09-11 are preserved as recorded.
+They predate the scoring correction in pre-push-safety-net task 1.3 and must not be
+interpreted as end-to-end detection or pre-push acceptance results.
+
+An audit on 2026-09-12 read all eight artifacts. Each records VAmPI-SQLI with
+`position: null`, `reached_transitively: true` and a repository `budget_at_recall`
+of 16. The old position instrument assigned that transitive label solely because
+it could not locate a matching region, then omitted the target from the budget
+calculation. These artifacts contain no queried witness establishing that label.
+This does not negate separately demonstrated VAmPI detections; it means this
+particular metric cannot establish them.
+
+Other recorded positions, scores, and elapsed stage costs remain historical
+observations under their recorded setup. Rank coverage is not detection recall,
+and a stage duration is not total hook latency. Use the corrected instrument and
+an explicit frozen profile for new comparisons. Retain unknown and unanswered
+cases, and require a queried target witness before counting transitive detection.
+Do not rewrite these files to make an earlier run appear to use the new semantics.

benchmarks/ranking/position.py

@@ -1,23 +1,19 @@
-"""Phase 0 of flow-aware-ranking: where does each pinned CVE sit in the order the scan actually examines?
+"""Positional coverage baseline v2: ranks are not detection recall.
 
-The ranker's success metric is not "is the CVE in the budget" -- on Paid Memberships Pro it already is, at
-position 390 of 4,463 -- but **how small the budget can be while every CVE stays in it**. That number is
-`budget_at_recall`, and until it is written down nothing later can claim to have improved it.
-
-This is the baseline. It runs no engine and asks no model: it is the region pipeline exactly as the scan
-uses it, sorted exactly as the scan sorts it, with each known vulnerability's position read off. Offline,
-against pinned checkouts, so two runs of one repository agree.
-
-    ousast-venv/bin/python benchmarks/ranking/position.py            # every pinned recipe with a checkout
-    ousast-venv/bin/python benchmarks/ranking/position.py --json     # the same, as an artifact
+Every declared target remains visible, including unmatched sites. No transitive detection
+is inferred here: that requires recorded answered queries and target witnesses in the
+push scorer. Historical v1 artifacts remain unchanged.
 """
 
 from __future__ import annotations
 
 import argparse
+import hashlib
 import json
 import os
+import subprocess
 import sys
+import time
 from collections import Counter
 from pathlib import Path
 
@@ -101,10 +97,55 @@ def evidence_ordered_regions(root: Path, regions: list[ScanRegion], language: st
 
 
 def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidence: bool = False, language: str = "") -> dict[str, object]:
-    regions = ordered_regions(root)
+    started = time.monotonic()
+    inputs = []
+    for path in sorted({str(item["file"]) for item in known}):
+        try:
+            content = (root / path).read_bytes()
+            if not content:
+                raise ValueError("empty source")
+            inputs.append(dict(path=path, bytes_read=len(content), sha256=hashlib.sha256(content).hexdigest()))
+        except (OSError, ValueError) as exc:
+            inputs.append(dict(path=path, bytes_read=0, problem=str(exc)))
     facts: dict[str, object] = {}
+    # Frozen identity recorded before the evidence pass; this is a rank-only experiment.
+    source_root = Path(__file__).resolve().parents[2]
+    version_paths = [
+        "benchmarks/ranking/position.py",
+        "src/openultrasast/model/regions.py",
+        "src/openultrasast/model/scan.py",
+        "src/openultrasast/model/evidence.py",
+        "src/openultrasast/model/layout.py",
+    ]
+    checkout = {}
+    try:
+        commit = (
+            subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], stderr=subprocess.DEVNULL, timeout=10).decode().strip()
+        )
+        status = subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], stderr=subprocess.DEVNULL, timeout=10)
+        checkout = dict(commit=commit, clean=not status, status_sha256=hashlib.sha256(status).hexdigest())
+    except (OSError, subprocess.SubprocessError):
+        checkout = dict(problem="checkout commit unavailable")
+    profile = dict(
+        checkout=checkout,
+        version="position-v2",
+        requested_ranking_mode="evidence" if by_evidence else "static",
+        known_targets=known,
+        input_witnesses=inputs,
+        implementation={path: hashlib.sha256((source_root / path).read_bytes()).hexdigest() for path in version_paths},
+        detection="not_measured",
+    )
+    profile["sha256"] = hashlib.sha256(json.dumps(profile, sort_keys=True).encode()).hexdigest()
+    regions = ordered_regions(root) if root.is_dir() else []
+    actual_mode = "static"
     if by_evidence:
-        regions, facts = evidence_ordered_regions(root, regions, language)
+        try:
+            regions, facts = evidence_ordered_regions(root, regions, language)
+            actual_mode = "evidence"
+        except (OSError, RuntimeError, subprocess.SubprocessError, SystemExit) as exc:
+            regions = []
+            actual_mode = "unavailable"
+            facts = dict(evidence_failure=str(exc))
     tiers = Counter(r.rank for r in regions)
     largest = max(tiers.values()) if tiers else 0
     rows: list[dict[str, object]] = []
@@ -112,7 +153,11 @@ def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidenc
         # A known vulnerability names a file and a function. Match on both, path by suffix because a recipe
         # writes it relative to the repository and a region may carry it the same way or deeper.
         hit = next(
-            (i for i, r in enumerate(regions) if r.path.endswith(str(item["file"])) and r.function == item["function"]),
+            (
+                i
+                for i, r in enumerate(regions)
+                if r.path in (str(item["file"]), str(root / str(item["file"]))) and r.function == item["function"]
+            ),
             None,
         )
         rows.append(
@@ -120,26 +165,39 @@ def measure(name: str, root: Path, known: list[dict[str, object]], *, by_evidenc
                 "id": item["id"],
                 "family": item["family"],
                 "site": f"{item['file']}:{item['line']}:{item['function']}",
-                # Zero-based. None means no region is named after this function: the site is a SINK that an
-                # entry-point region elsewhere reaches through `callDepth` -- VAmPI's SQL injection lives in
-                # `models/user_model.py:get_user` and is found from the handler that calls it. Its true
-                # position is that handler's, which only the flow can say, so it is reported and left out of
-                # `budget_at_recall` rather than counted as a miss it is not.
+                # Zero-based position is only a region match, never a completed detection.
                 "position": hit,
                 "rank": regions[hit].rank if hit is not None else None,
                 "in_scope": item["in_scope"],
-                "reached_transitively": hit is None,
+                "reached_transitively": False,
+                "detection": "not_measured",
+                "position_status": "matched" if hit is not None else "unresolved_target",
             }
         )
     positioned = [r["position"] for r in rows if r["position"] is not None and r["in_scope"]]
+    supported = [row for row in rows if row["in_scope"]]
+    all_located = bool(supported) and len(positioned) == len(supported)
     return {
+        "schema_version": 2,
+        "measurement": "positional_coverage_not_detection",
+        "ranking_mode": actual_mode,
+        "profile": profile,
+        "detection_recall": None,
+        "supported_population": len(supported),
+        "unmatched_supported": len(supported) - len(positioned),
+        "input_witnesses": inputs,
+        "input_status": "readable" if inputs and all(row["bytes_read"] for row in inputs) else "incomplete",
+        "total_seconds": round(time.monotonic() - started, 3),
         "repo": name,
         "regions": len(regions),
         "distinct_ranks": len(tiers),
         "largest_tier_share": round(largest / len(regions), 3) if regions else 0.0,
         "largest_tier_rank": max(tiers, key=lambda k: tiers[k]) if tiers else None,
         # The smallest examined-region budget that still holds every in-scope known vulnerability.
-        "budget_at_recall": (max(positioned) + 1) if positioned else None,
+        "budget_at_recall": None,  # Retired misleading name: rank does not measure recall.
+        "budget_at_full_target_position_coverage": (max(positioned) + 1)
+        if all_located and all(row["bytes_read"] for row in inputs)
+        else None,
         "known": rows,
         **facts,
     }
@@ -160,8 +218,7 @@ def main() -> int:
             continue
         root = checkout_path(recipe)
         if not root.is_dir():
-            print(f"{recipe.name}: not checked out, skipping", file=sys.stderr)
-            continue
+            print(f"{recipe.name}: not checked out; retaining unresolved targets", file=sys.stderr)
         known = [
             {"id": k.id, "family": k.family, "file": k.file, "function": k.function, "line": k.line, "in_scope": k.in_scope}
             for k in recipe.known
@@ -173,20 +230,20 @@ def main() -> int:
 
     if args.json:
         print(json.dumps(results, indent=2))
-        return 0
+        return 1 if any(r["unmatched_supported"] or r["input_status"] != "readable" or r.get("evidence_failure") for r in results) else 0
     for r in results:
         print(
             f"{r['repo']}: {r['regions']} regions, {r['distinct_ranks']} distinct ranks, "
             f"{100 * float(r['largest_tier_share']):.1f}% share rank {r['largest_tier_rank']}"
         )
         for k in r["known"]:
-            where = "reached via another region" if k["position"] is None else f"position {k['position']} (rank {k['rank']})"
+            where = "unresolved target position" if k["position"] is None else f"position {k['position']} (rank {k['rank']})"
             scope = "" if k["in_scope"] else "  [out of scope]"
             print(f"   {k['id']:16} {where:28} {k['site']}{scope}")
-        print(f"   budget_at_recall = {r['budget_at_recall']}")
+        print(f"   budget_at_full_target_position_coverage = {r['budget_at_full_target_position_coverage']}; detection not measured")
         if "tier_counts" in r:
             print(f"   build {r['build_seconds']}s, evidence {r['evidence_seconds']}s over {r['pairs']} pairs, tiers {r['tier_counts']}")
-    return 0
+    return 1 if any(r["unmatched_supported"] or r["input_status"] != "readable" or r.get("evidence_failure") for r in results) else 0
 
 
 if __name__ == "__main__":