feat(benchmark): freeze the valid-benchmark pilot, 12 verified positives + 240 negatives

First live run of the valid detection benchmark. Labels come from the fix
site (fix^1), never SZZ.

- 32 held-out Python/JS advisories sourced from the public pool-p1 set,
  CWE + text from OSV. 133 candidate locations -> 125 after dropping
  tests/minified.
- 125 DeepSeek verification calls: 31 locations passed all three checks
  (vulnerable pre-fix, fix addresses it, reachable in-repo) across 12 repos.
- Maintainer spot-check accepted all 12.
- 20 ordinary negatives per positive change from the same repos, 240 total.
- Held-out guard: 0 dropped against the training-only used set.
- Frozen manifest at benchmarks/valid/pilot/ (252 changes), provenance and
  NOTES under benchmarks/measurements/2026-10-12-valid-benchmark-pilot/.

Three fixes found by the live run:
- catalog.py: skip a site the fix renamed or removed instead of aborting.
- assemble.py: negative ratio counts positive changes, not accepted
  functions (malla would have demanded 260 negatives). Test updated.
- held-out guard: run against training sources only; build_used() also
  scans measurements/populations, which flagged the 12 repos via my own
  SZZ-pool analysis outputs (circular).

Replay and metrics not yet run; see NOTES.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

benchmarks/valid/assemble.py

@@ -77,7 +77,11 @@ def freeze(
         head = (history_heads or {}).get(repo) or git.run("rev-parse", "HEAD").strip()
         heads[repo] = head
         history = sorted(extract.ordinary_history(git, head), key=lambda row: row["head"])
-        ordinary, exclusions = negatives(history, fixes, functions, count=ratio * len(positives), seed=seed)
+        # Ratio is per positive CHANGE (a fix commit), not per accepted function. Several
+        # accepted functions in one fix collapse to one vulnerability change below, so the
+        # negative count must track distinct (commit, parent) pairs.
+        change_count = len({(case["commit"], case["parent"]) for case in positives})
+        ordinary, exclusions = negatives(history, fixes, functions, count=ratio * change_count, seed=seed)
         draws[repo] = exclusions
         changes = []
         pairs = defaultdict(list)

benchmarks/valid/catalog.py

@@ -103,7 +103,13 @@ def build_cases(git, source, *, cwe: str | None = None, advisory_text: str = "")
             function=site["function"],
             method="fix_site",
         )
-        sides = read_functions(git, case)
+        try:
+            sides = read_functions(git, case)
+        except extract.InstrumentFailure:
+            # The fix renamed, moved, or removed the function, so there is no clean
+            # pre/post pair at this site. Skip it during enumeration. Verification
+            # stays strict: a verdict on an unread case is still rejected there.
+            continue
         if extract.normalized(sides["pre_fix"]) == extract.normalized(sides["post_fix"]):
             continue
         case["function_lines"] = [sides["pre_fix_lines"]]

benchmarks/valid/pilot/freeze-valid.json

@@ -0,0 +1,165 @@
+{
+  "catalog_digest": "3f641f15619a54351ae5032fb06cc97f37d5876697f3b71bb966a3c1f539d3e4",
+  "dropped_count": 0,
+  "freeze_digest": "c866c0e812c0b54476c7168fdb0e4a20652de0d30485270afecbe51e5c35e424",
+  "guard_result": 0,
+  "history_heads": {
+    "bytebase/dbhub": "f7b31468be4ff0c70b1c662b3e535fe36f4dad06",
+    "contentful/contentful-mcp-server": "edb981b6e2a75173ff57e62ca344bc76a5eec766",
+    "fastapi-users/fastapi-users": "e9c57ff7ff2fd385bb0f6f795b524e9736b64290",
+    "i18next/i18next-http-middleware": "824bdfb49e777a499a13ea19fa70a07f8fe68ad2",
+    "imzbf/md-editor-v3": "1a6f4c61ebf38a836cc985b786f39a126441a835",
+    "jupyter-lsp/jupyterlab-lsp": "e2e569321bc45702718030fabfe7dd38d0457f25",
+    "matrix-org/matrix-appservice-bridge": "6392ab0550e23fffa9f014411148c667ba2e77da",
+    "nvbn/thefuck": "c7e7e1d884d3bb241ea6448f72a989434c2a35ec",
+    "ondata/ckan-mcp-server": "d7a24143699c64c4dcea0ca7041a19fc875f8cef",
+    "webpack/webpack-bundle-analyzer": "6046503c6e183cc2bcf069a7d58dcf87f4870f85",
+    "zenitram/malla": "97d141888677008e856a833b68199169efa6c828",
+    "zereight/gitlab-mcp": "3a5dff69ed7df5a0d1c3af388714f188ff2dc4af"
+  },
+  "label_check_counts": {
+    "confirmed": 31,
+    "refactor": 0,
+    "unchecked": 0,
+    "unclear": 0
+  },
+  "negative_ratio": 20,
+  "ordinary_exclusions": {
+    "bytebase/dbhub": {
+      "advisory_fix": 1,
+      "dependency": 13,
+      "eligible": 273,
+      "manifest_only": 54,
+      "merge_or_root": 19,
+      "no_source": 210,
+      "revert": 3,
+      "security": 26,
+      "size_cap": 12,
+      "vulnerable_function": 3
+    },
+    "contentful/contentful-mcp-server": {
+      "advisory_fix": 1,
+      "dependency": 202,
+      "eligible": 53,
+      "manifest_only": 1,
+      "merge_or_root": 14,
+      "no_source": 41,
+      "revert": 2,
+      "security": 12,
+      "size_cap": 10,
+      "vulnerable_function": 10
+    },
+    "fastapi-users/fastapi-users": {
+      "advisory_fix": 1,
+      "dependency": 346,
+      "eligible": 171,
+      "manifest_only": 18,
+      "merge_or_root": 19,
+      "no_source": 154,
+      "revert": 4,
+      "security": 17,
+      "size_cap": 2,
+      "vulnerable_function": 31
+    },
+    "i18next/i18next-http-middleware": {
+      "advisory_fix": 1,
+      "dependency": 47,
+      "eligible": 70,
+      "manifest_only": 74,
+      "merge_or_root": 4,
+      "no_source": 43,
+      "security": 8,
+      "vulnerable_function": 15
+    },
+    "imzbf/md-editor-v3": {
+      "advisory_fix": 1,
+      "dependency": 7,
+      "eligible": 576,
+      "manifest_only": 226,
+      "merge_or_root": 100,
+      "no_source": 161,
+      "revert": 7,
+      "security": 12,
+      "size_cap": 6,
+      "vulnerable_function": 51
+    },
+    "jupyter-lsp/jupyterlab-lsp": {
+      "advisory_fix": 1,
+      "dependency": 36,
+      "eligible": 131,
+      "manifest_only": 19,
+      "merge_or_root": 383,
+      "no_source": 90,
+      "size_cap": 5,
+      "vulnerable_function": 2
+    },
+    "matrix-org/matrix-appservice-bridge": {
+      "dependency": 18,
+      "eligible": 172,
+      "manifest_only": 10,
+      "merge_or_root": 235,
+      "no_source": 141,
+      "security": 4,
+      "size_cap": 3
+    },
+    "nvbn/thefuck": {
+      "advisory_fix": 1,
+      "dependency": 76,
+      "eligible": 429,
+      "manifest_only": 10,
+      "merge_or_root": 299,
+      "no_source": 114,
+      "revert": 3,
+      "security": 4,
+      "vulnerable_function": 1
+    },
+    "ondata/ckan-mcp-server": {
+      "advisory_fix": 1,
+      "dependency": 681,
+      "eligible": 72,
+      "manifest_only": 4,
+      "merge_or_root": 10,
+      "no_source": 310,
+      "revert": 2,
+      "security": 39,
+      "size_cap": 5,
+      "vulnerable_function": 71
+    },
+    "webpack/webpack-bundle-analyzer": {
+      "dependency": 43,
+      "eligible": 82,
+      "manifest_only": 18,
+      "merge_or_root": 104,
+      "no_source": 131,
+      "revert": 2,
+      "security": 4,
+      "size_cap": 1,
+      "vulnerable_function": 18
+    },
+    "zenitram/malla": {
+      "advisory_fix": 1,
+      "dependency": 5,
+      "eligible": 76,
+      "manifest_only": 1,
+      "merge_or_root": 1,
+      "no_source": 25,
+      "security": 8,
+      "size_cap": 3,
+      "vulnerable_function": 7
+    },
+    "zereight/gitlab-mcp": {
+      "advisory_fix": 1,
+      "dependency": 77,
+      "eligible": 170,
+      "manifest_only": 41,
+      "merge_or_root": 280,
+      "no_source": 211,
+      "revert": 3,
+      "security": 16,
+      "size_cap": 7,
+      "vulnerable_function": 8
+    }
+  },
+  "used_set_digest": "fb7b0cc1f38bc2fb513123d8b4a0e16afd5501b42ef1a49ad7ecedd8eba55885",
+  "verification_digest": "9363de4b7466953ad72cb29f3e91a01b3cfc8411b35310e0fa270f967836b41b"
+}