measure: the candidate generator must be the IR, not the ruleset

A candidate-driven detector can only judge sites something else enumerates,
so the enumerator caps recall. Measured over the 58 scorable web pairs before
building anything:

  pattern ruleset:  12.1% ceiling -- and the misses are not near-misses, the
                    ruleset finds nothing at all in most of these files
  semantic IR:      96.6% ceiling, a median of ten call sites and bindings
                    inside the labeled function

The ruleset would have capped us at 12.1% against the 64.9% the unconstrained
loop already reaches. That design was one commit from being built.

Two corpus defects found on the way, both mine from task 4.1. The overleaf
excerpt started mid-import, because `_declarator_start` walks back to the
nearest `}` and Overleaf writes JavaScript without semicolons, so the nearest
one was inside `import { expressify }`. It then ended mid-object-literal,
because registration mentions were appended as bare lines. Neither parsed,
and a pair that cannot be parsed cannot be scored. Registration mentions now
expand to the statement that contains them; the ceiling moved 94.8% -> 96.6%.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

benchmarks/measurements/2026-09-06-candidate-ceiling.json

@@ -0,0 +1,45 @@
+{
+  "baseline_for_comparison": {
+    "detection_rate": 0.649,
+    "leak_rate": 0.123,
+    "note": "deepseek-v4-flash, K=5, unconstrained tool loop, 57 scorable pairs across vibe-py and agent-vfc",
+    "pair_correct": 0.526
+  },
+  "candidate_sources": {
+    "pattern_ruleset (quick_scan_findings)": {
+      "by_family": {
+        "access_control": 0.0,
+        "config_secrets": 0.0,
+        "deserialization": 0.5,
+        "injection": 0.27,
+        "output_encoding": 0.0,
+        "path": 0.0,
+        "prototype": 0.0,
+        "untrusted_destination": 0.0
+      },
+      "ceiling": 0.121,
+      "note": "the ruleset finds nothing at all in most of these files, not merely nothing inside the labeled function",
+      "pairs": 58,
+      "pairs_with_a_candidate": 7
+    },
+    "semantic_ir (call sites + bindings)": {
+      "ceiling": 0.966,
+      "gaps": [
+        "edutrace-page-ee7ba2 (labeled function absent from the IR)",
+        "money-track-category-icons-4d4a5b (no calls or bindings in the labeled function)"
+      ],
+      "pairs": 58,
+      "pairs_with_a_candidate": 56,
+      "sites_per_labeled_function": {
+        "max": 33,
+        "median": 10,
+        "min": 1,
+        "p90": 22
+      }
+    }
+  },
+  "command": "offline: parse each vulnerable side, count call sites and bindings inside the labeled function",
+  "conclusion": "The candidate generator must be the semantic IR, not the pattern ruleset. The ruleset would cap recall at 12.1%, well below the 64.9% the unconstrained loop already reaches; the IR caps it at 96.6% and hands the model a median of ten sites to judge instead of a repository to search.",
+  "generated": "2026-09-06",
+  "note": "A candidate-driven detector can only judge sites something else enumerates, so the enumerator sets a hard ceiling on recall. Measured over the 58 scorable vendored web pairs, before building anything."
+}

benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/fixed.tsx

@@ -64,8 +64,27 @@
     }
   }
 
+        try {
           await loadNamespaces(true)
+          await loadData()
+        } catch (err) {
+          console.error('Error loading namespaces:', err)
+        }
+        try {
           await loadNamespaces(true)
+          // Always load data after namespaces are loaded (defaults to 'all' to show all resources)
+          await loadData()
+        } catch (err) {
+          console.error('Error loading namespaces after connection:', err)
+          setErrorMessage('✅ Connected, but failed to load namespaces. Click "Load Namespaces" to retry.')
+        }
+      if (apiUrl && !deployedNamespace) {
         loadNamespaces(true)
+      }
+    if (apiUrl && !isConnected && !isConnecting) {
+      console.log('🚀 Auto-connecting to Kubernetes cluster via ServiceAccount...')
       loadNamespaces(true) // Skip connection check since we're auto-connecting
+    }
+    if (apiUrl && !isConnected) {
       loadNamespaces(true);
+    }

benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/vuln.tsx

@@ -62,8 +62,27 @@
     }
   }
 
+        try {
           await loadNamespaces(true)
+          await loadData()
+        } catch (err) {
+          console.error('Error loading namespaces:', err)
+        }
+        try {
           await loadNamespaces(true)
+          // Always load data after namespaces are loaded (defaults to 'all' to show all resources)
+          await loadData()
+        } catch (err) {
+          console.error('Error loading namespaces after connection:', err)
+          setErrorMessage('✅ Connected, but failed to load namespaces. Click "Load Namespaces" to retry.')
+        }
+      if (apiUrl && !deployedNamespace) {
         loadNamespaces(true)
+      }
+    if (apiUrl && !isConnected && !isConnecting) {
+      console.log('🚀 Auto-connecting to Kubernetes cluster via ServiceAccount...')
       loadNamespaces(true) // Skip connection check since we're auto-connecting
+    }
+    if (apiUrl && !isConnected) {
       loadNamespaces(true);
+    }

benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/fixed.mjs

@@ -9,13 +9,7 @@
 // relpath: services/web/app/src/Features/Exports/ExportsController.mjs
 // provenance: agent
 // mechanism: identity_from_request_body
-// upstream_start: 1
-
-from '@overleaf/promise-utils'
-import SessionManager from '../Authentication/SessionManager.mjs'
-import logger from '@overleaf/logger'
-import OError from '@overleaf/o-error'
-import settings from '@overleaf/settings'
+// upstream_start: 8
 
 async function exportProject(req, res, next) {
   const { project_id: projectId, brand_variation_id: brandVariationId } =
@@ -82,4 +76,8 @@ async function exportProject(req, res, next) {
   }
 }
 
+export default {
+  exportDownload: expressify(exportDownload),
   exportProject: expressify(exportProject),
+  exportStatus: expressify(exportStatus),
+}

benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/vuln.mjs

@@ -9,12 +9,7 @@
 // relpath: services/web/app/src/Features/Exports/ExportsController.mjs
 // provenance: agent
 // mechanism: identity_from_request_body
-// upstream_start: 1
-
-from '@overleaf/promise-utils'
-import SessionManager from '../Authentication/SessionManager.mjs'
-import logger from '@overleaf/logger'
-import OError from '@overleaf/o-error'
+// upstream_start: 7
 
 async function exportProject(req, res, next) {
   const { project_id: projectId, brand_variation_id: brandVariationId } =
@@ -80,4 +75,8 @@ async function exportProject(req, res, next) {
   }
 }
 
+export default {
+  exportDownload: expressify(exportDownload),
   exportProject: expressify(exportProject),
+  exportStatus: expressify(exportStatus),
+}

benchmarks/pairs/harvest.py

@@ -64,6 +64,8 @@ _DEF_RE = re.compile(r"^(\s*)(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(")
 _CLASS_RE = re.compile(r"^(\s*)class\s+([A-Za-z_]\w*)\s*[(:]")
 _COMMENTED_CODE = re.compile(r"^#\s*(?:@|(?:async\s+)?def\s|class\s|function\s)")
 _CONTROL_HEAD = re.compile(r"^(?:else\s+)?(?:if|for|while|switch|catch|do|try|finally)\b")
+# A module-level import or its continuation: `import ... from 'x'`, `from 'x'`, `export ... from 'x'`.
+_IMPORT_LINE = re.compile(r"^(?:import\b|export\b.*\bfrom\b|from\s+[\'\"])")
 _INDENTED = {"python", "yaml"}
 
 
@@ -270,16 +272,78 @@ def extract_handler_context(source: str, function_name: str, language: str = "c"
     mention = re.compile(rf"(?<![\w.]){re.escape(function_name)}(?!\w)")
     declaration = re.compile(rf"^\s*(?:async\s+)?(?:def|function|sub)\s+{re.escape(function_name)}\b")
     covered = set(range(start, end)) | set(header)
-    registrations = [
-        lines[index]
+    hits = [
+        index
         for index, masked in enumerate(masked_lines)
         if index not in covered and index < len(lines) and mention.search(masked) and not declaration.match(masked)
     ]
+    registrations = _registration_statements(source, lines, hits, language, covered)
     if header:
         body = "\n".join(lines[index] for index in header) + "\n" + body
     if not registrations:
         return body
-    return body.rstrip("\n") + "\n\n" + "\n".join(line.strip() for line in registrations) + "\n"
+    return body.rstrip("\n") + "\n\n" + "\n".join(registrations) + "\n"
+
+
+def _registration_statements(source: str, lines: Sequence[str], hits: Sequence[int], language: str, covered: set[int]) -> list[str]:
+    """Each registration mention expanded to the statement that contains it, deduplicated and in order.
+
+    A bare line is not always a statement: `export default { exportProject: expressify(exportProject) }` mentions
+    the handler inside an object literal, and appending that one line leaves the excerpt unparsable — which costs
+    the pair its function ranges and drops it out of every denominator."""
+    if language in _INDENTED:
+        return [lines[index].strip() for index in hits]
+    out: list[str] = []
+    taken: set[int] = set(covered)
+    for index in hits:
+        if index in taken:
+            continue
+        span = _statement_span(source, lines, index)
+        if span is None:
+            taken.add(index)
+            out.append(lines[index].strip())
+            continue
+        first, last = span
+        if any(position in taken for position in range(first, last + 1)):
+            continue
+        taken.update(range(first, last + 1))
+        out.append("\n".join(lines[first : last + 1]).rstrip())
+    return out
+
+
+def _statement_span(source: str, lines: Sequence[str], index: int) -> tuple[int, int] | None:
+    """The line range of the smallest balanced `(...)`/`{...}`/`[...]` construct enclosing ``index``, with the
+    line that opens it; None when the line already stands on its own."""
+    masked_lines = mask_comments_and_strings(source, "javascript").splitlines()
+    depth = 0
+    first = index
+    for position in range(index, -1, -1):
+        for char in reversed(masked_lines[position] if position < len(masked_lines) else ""):
+            if char in ")}]":
+                depth += 1
+            elif char in "({[":
+                if depth == 0:
+                    first = position
+                    break
+                depth -= 1
+        else:
+            continue
+        break
+    else:
+        return None
+    if first == index and _balanced(masked_lines[index] if index < len(masked_lines) else ""):
+        return None
+    depth = 0
+    for last in range(first, len(lines)):
+        line = masked_lines[last] if last < len(masked_lines) else ""
+        depth += sum(line.count(char) for char in "({[") - sum(line.count(char) for char in ")}]")
+        if last >= index and depth <= 0:
+            return first, last
+    return first, len(lines) - 1
+
+
+def _balanced(line: str) -> bool:
+    return sum(line.count(char) for char in "({[") == sum(line.count(char) for char in ")}]")
 
 
 def _indent(line: str) -> int:
@@ -496,12 +560,15 @@ def _declarator_start(source: str, masked: str, anchor: int) -> int:
                 break
         pos -= 1
     start = pos + 1
-    # Skip blank and comment-only lines between the boundary and the declarator.
+    # Skip blank and comment-only lines between the boundary and the declarator, and any import or export
+    # statement the boundary landed inside. JavaScript written without semicolons puts the nearest `}` in
+    # `import { x } from '...'`, so the walk-back stops mid-statement and the excerpt does not parse at all.
     while True:
         line_end = source.find("\n", start)
         if line_end < 0:
             break
-        if masked[start:line_end].strip():
+        line = masked[start:line_end].strip()
+        if line and not _IMPORT_LINE.match(line):
             break
         start = line_end + 1
     return start

tests/test_pair_registration_context.py

@@ -425,3 +425,32 @@ def test_prose_inside_a_decorator_never_grants_a_guard(tmp_path: Path) -> None:
     _, targets = preprocess_repository(tmp_path)
     access = {record.function_name: record.access_level for record in analyze_entry_points(tmp_path, targets) if record.function_name}
     assert access == {"delete": "public", "get": "authenticated"}
+
+
+def test_a_javascript_excerpt_never_starts_in_the_middle_of_an_import() -> None:
+    """`_declarator_start` walks back to the previous `;` or `}`. In JavaScript written without semicolons the
+    nearest `}` is the one in `import { expressify } from ...`, so the excerpt began mid-statement and the file
+    did not parse at all — a pair that cannot be parsed cannot be scored."""
+    lib = _lib()
+    source = (
+        "import { expressify } from '@overleaf/promise-utils'\n"
+        "import SessionManager from '../Authentication/SessionManager.mjs'\n\n"
+        "async function exportProject(req, res, next) {\n"
+        "  return res.send(req.params.project_id)\n"
+        "}\n"
+    )
+    excerpt = lib["extract_function"](source, "exportProject", "javascript")
+    assert excerpt.startswith("async function exportProject("), excerpt.splitlines()[0]
+    assert "from '@overleaf" not in excerpt
+
+
+def test_the_overleaf_excerpt_parses_on_both_sides() -> None:
+    from openultrasast.semantic.ir import parse_file
+
+    case = next(
+        item for item in select_vendored(load_pair_catalog(DEFAULT_CATALOG)) if item.name == "overleaf-with-claude-exportscontroller-3980b9"
+    )
+    for path in (case.vuln_file, case.fixed_file):
+        ir = parse_file(case.relpath, path.read_text(), "javascript")
+        assert ir.parse_ok, f"{path.name}: {ir.reason}"
+        assert any(function.name == "exportProject" for function in ir.functions)