From 309faad266a1a5b3752d1c930ccbd69f4b52199b Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 6 Sep 2026 16:04:07 +0200 Subject: [PATCH] measure: the candidate generator must be the IR, not the ruleset A candidate-driven detector can only judge sites something else enumerates, so the enumerator caps recall. Measured over the 58 scorable web pairs before building anything: pattern ruleset: 12.1% ceiling -- and the misses are not near-misses, the ruleset finds nothing at all in most of these files semantic IR: 96.6% ceiling, a median of ten call sites and bindings inside the labeled function The ruleset would have capped us at 12.1% against the 64.9% the unconstrained loop already reaches. That design was one commit from being built. Two corpus defects found on the way, both mine from task 4.1. The overleaf excerpt started mid-import, because `_declarator_start` walks back to the nearest `}` and Overleaf writes JavaScript without semicolons, so the nearest one was inside `import { expressify }`. It then ended mid-object-literal, because registration mentions were appended as bare lines. Neither parsed, and a pair that cannot be parsed cannot be scored. Registration mentions now expand to the statement that contains them; the ceiling moved 94.8% -> 96.6%. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .../2026-09-06-candidate-ceiling.json | 45 +++++++++++ .../fixed.tsx | 29 +++++-- .../vuln.tsx | 29 +++++-- .../fixed.mjs | 14 ++-- .../vuln.mjs | 13 ++-- benchmarks/pairs/harvest.py | 77 +++++++++++++++++-- tests/test_pair_registration_context.py | 29 +++++++ 7 files changed, 206 insertions(+), 30 deletions(-) create mode 100644 benchmarks/measurements/2026-09-06-candidate-ceiling.json diff --git a/benchmarks/measurements/2026-09-06-candidate-ceiling.json b/benchmarks/measurements/2026-09-06-candidate-ceiling.json new file mode 100644 index 0000000..4c51fc3 --- /dev/null +++ b/benchmarks/measurements/2026-09-06-candidate-ceiling.json @@ -0,0 +1,45 @@ +{ + "baseline_for_comparison": { + "detection_rate": 0.649, + "leak_rate": 0.123, + "note": "deepseek-v4-flash, K=5, unconstrained tool loop, 57 scorable pairs across vibe-py and agent-vfc", + "pair_correct": 0.526 + }, + "candidate_sources": { + "pattern_ruleset (quick_scan_findings)": { + "by_family": { + "access_control": 0.0, + "config_secrets": 0.0, + "deserialization": 0.5, + "injection": 0.27, + "output_encoding": 0.0, + "path": 0.0, + "prototype": 0.0, + "untrusted_destination": 0.0 + }, + "ceiling": 0.121, + "note": "the ruleset finds nothing at all in most of these files, not merely nothing inside the labeled function", + "pairs": 58, + "pairs_with_a_candidate": 7 + }, + "semantic_ir (call sites + bindings)": { + "ceiling": 0.966, + "gaps": [ + "edutrace-page-ee7ba2 (labeled function absent from the IR)", + "money-track-category-icons-4d4a5b (no calls or bindings in the labeled function)" + ], + "pairs": 58, + "pairs_with_a_candidate": 56, + "sites_per_labeled_function": { + "max": 33, + "median": 10, + "min": 1, + "p90": 22 + } + } + }, + "command": "offline: parse each vulnerable side, count call sites and bindings inside the labeled function", + "conclusion": "The candidate generator must be the semantic IR, not the pattern ruleset. The ruleset would cap recall at 12.1%, well below the 64.9% the unconstrained loop already reaches; the IR caps it at 96.6% and hands the model a median of ten sites to judge instead of a repository to search.", + "generated": "2026-09-06", + "note": "A candidate-driven detector can only judge sites something else enumerates, so the enumerator sets a hard ceiling on recall. Measured over the 58 scorable vendored web pairs, before building anything." +} diff --git a/benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/fixed.tsx b/benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/fixed.tsx index c7ee13f..5ca6ecf 100644 --- a/benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/fixed.tsx +++ b/benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/fixed.tsx @@ -64,8 +64,27 @@ } } -await loadNamespaces(true) -await loadNamespaces(true) -loadNamespaces(true) -loadNamespaces(true) // Skip connection check since we're auto-connecting -loadNamespaces(true); + try { + await loadNamespaces(true) + await loadData() + } catch (err) { + console.error('Error loading namespaces:', err) + } + try { + await loadNamespaces(true) + // Always load data after namespaces are loaded (defaults to 'all' to show all resources) + await loadData() + } catch (err) { + console.error('Error loading namespaces after connection:', err) + setErrorMessage('✅ Connected, but failed to load namespaces. Click "Load Namespaces" to retry.') + } + if (apiUrl && !deployedNamespace) { + loadNamespaces(true) + } + if (apiUrl && !isConnected && !isConnecting) { + console.log('🚀 Auto-connecting to Kubernetes cluster via ServiceAccount...') + loadNamespaces(true) // Skip connection check since we're auto-connecting + } + if (apiUrl && !isConnected) { + loadNamespaces(true); + } diff --git a/benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/vuln.tsx b/benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/vuln.tsx index e172068..691b86e 100644 --- a/benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/vuln.tsx +++ b/benchmarks/pairs/agent-vfc/lensops-kubernetesmanagement-fe4910/vuln.tsx @@ -62,8 +62,27 @@ } } -await loadNamespaces(true) -await loadNamespaces(true) -loadNamespaces(true) -loadNamespaces(true) // Skip connection check since we're auto-connecting -loadNamespaces(true); + try { + await loadNamespaces(true) + await loadData() + } catch (err) { + console.error('Error loading namespaces:', err) + } + try { + await loadNamespaces(true) + // Always load data after namespaces are loaded (defaults to 'all' to show all resources) + await loadData() + } catch (err) { + console.error('Error loading namespaces after connection:', err) + setErrorMessage('✅ Connected, but failed to load namespaces. Click "Load Namespaces" to retry.') + } + if (apiUrl && !deployedNamespace) { + loadNamespaces(true) + } + if (apiUrl && !isConnected && !isConnecting) { + console.log('🚀 Auto-connecting to Kubernetes cluster via ServiceAccount...') + loadNamespaces(true) // Skip connection check since we're auto-connecting + } + if (apiUrl && !isConnected) { + loadNamespaces(true); + } diff --git a/benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/fixed.mjs b/benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/fixed.mjs index f7a580c..e8ae8ab 100644 --- a/benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/fixed.mjs +++ b/benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/fixed.mjs @@ -9,13 +9,7 @@ // relpath: services/web/app/src/Features/Exports/ExportsController.mjs // provenance: agent // mechanism: identity_from_request_body -// upstream_start: 1 - -from '@overleaf/promise-utils' -import SessionManager from '../Authentication/SessionManager.mjs' -import logger from '@overleaf/logger' -import OError from '@overleaf/o-error' -import settings from '@overleaf/settings' +// upstream_start: 8 async function exportProject(req, res, next) { const { project_id: projectId, brand_variation_id: brandVariationId } = @@ -82,4 +76,8 @@ async function exportProject(req, res, next) { } } -exportProject: expressify(exportProject), +export default { + exportDownload: expressify(exportDownload), + exportProject: expressify(exportProject), + exportStatus: expressify(exportStatus), +} diff --git a/benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/vuln.mjs b/benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/vuln.mjs index f9eb6c2..ec3c86a 100644 --- a/benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/vuln.mjs +++ b/benchmarks/pairs/agent-vfc/overleaf-with-claude-exportscontroller-3980b9/vuln.mjs @@ -9,12 +9,7 @@ // relpath: services/web/app/src/Features/Exports/ExportsController.mjs // provenance: agent // mechanism: identity_from_request_body -// upstream_start: 1 - -from '@overleaf/promise-utils' -import SessionManager from '../Authentication/SessionManager.mjs' -import logger from '@overleaf/logger' -import OError from '@overleaf/o-error' +// upstream_start: 7 async function exportProject(req, res, next) { const { project_id: projectId, brand_variation_id: brandVariationId } = @@ -80,4 +75,8 @@ async function exportProject(req, res, next) { } } -exportProject: expressify(exportProject), +export default { + exportDownload: expressify(exportDownload), + exportProject: expressify(exportProject), + exportStatus: expressify(exportStatus), +} diff --git a/benchmarks/pairs/harvest.py b/benchmarks/pairs/harvest.py index 8bdd8cf..783df0f 100644 --- a/benchmarks/pairs/harvest.py +++ b/benchmarks/pairs/harvest.py @@ -64,6 +64,8 @@ _DEF_RE = re.compile(r"^(\s*)(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(") _CLASS_RE = re.compile(r"^(\s*)class\s+([A-Za-z_]\w*)\s*[(:]") _COMMENTED_CODE = re.compile(r"^#\s*(?:@|(?:async\s+)?def\s|class\s|function\s)") _CONTROL_HEAD = re.compile(r"^(?:else\s+)?(?:if|for|while|switch|catch|do|try|finally)\b") +# A module-level import or its continuation: `import ... from 'x'`, `from 'x'`, `export ... from 'x'`. +_IMPORT_LINE = re.compile(r"^(?:import\b|export\b.*\bfrom\b|from\s+[\'\"])") _INDENTED = {"python", "yaml"} @@ -270,16 +272,78 @@ def extract_handler_context(source: str, function_name: str, language: str = "c" mention = re.compile(rf"(? list[str]: + """Each registration mention expanded to the statement that contains it, deduplicated and in order. + + A bare line is not always a statement: `export default { exportProject: expressify(exportProject) }` mentions + the handler inside an object literal, and appending that one line leaves the excerpt unparsable — which costs + the pair its function ranges and drops it out of every denominator.""" + if language in _INDENTED: + return [lines[index].strip() for index in hits] + out: list[str] = [] + taken: set[int] = set(covered) + for index in hits: + if index in taken: + continue + span = _statement_span(source, lines, index) + if span is None: + taken.add(index) + out.append(lines[index].strip()) + continue + first, last = span + if any(position in taken for position in range(first, last + 1)): + continue + taken.update(range(first, last + 1)) + out.append("\n".join(lines[first : last + 1]).rstrip()) + return out + + +def _statement_span(source: str, lines: Sequence[str], index: int) -> tuple[int, int] | None: + """The line range of the smallest balanced `(...)`/`{...}`/`[...]` construct enclosing ``index``, with the + line that opens it; None when the line already stands on its own.""" + masked_lines = mask_comments_and_strings(source, "javascript").splitlines() + depth = 0 + first = index + for position in range(index, -1, -1): + for char in reversed(masked_lines[position] if position < len(masked_lines) else ""): + if char in ")}]": + depth += 1 + elif char in "({[": + if depth == 0: + first = position + break + depth -= 1 + else: + continue + break + else: + return None + if first == index and _balanced(masked_lines[index] if index < len(masked_lines) else ""): + return None + depth = 0 + for last in range(first, len(lines)): + line = masked_lines[last] if last < len(masked_lines) else "" + depth += sum(line.count(char) for char in "({[") - sum(line.count(char) for char in ")}]") + if last >= index and depth <= 0: + return first, last + return first, len(lines) - 1 + + +def _balanced(line: str) -> bool: + return sum(line.count(char) for char in "({[") == sum(line.count(char) for char in ")}]") def _indent(line: str) -> int: @@ -496,12 +560,15 @@ def _declarator_start(source: str, masked: str, anchor: int) -> int: break pos -= 1 start = pos + 1 - # Skip blank and comment-only lines between the boundary and the declarator. + # Skip blank and comment-only lines between the boundary and the declarator, and any import or export + # statement the boundary landed inside. JavaScript written without semicolons puts the nearest `}` in + # `import { x } from '...'`, so the walk-back stops mid-statement and the excerpt does not parse at all. while True: line_end = source.find("\n", start) if line_end < 0: break - if masked[start:line_end].strip(): + line = masked[start:line_end].strip() + if line and not _IMPORT_LINE.match(line): break start = line_end + 1 return start diff --git a/tests/test_pair_registration_context.py b/tests/test_pair_registration_context.py index be62aa3..b1d306a 100644 --- a/tests/test_pair_registration_context.py +++ b/tests/test_pair_registration_context.py @@ -425,3 +425,32 @@ def test_prose_inside_a_decorator_never_grants_a_guard(tmp_path: Path) -> None: _, targets = preprocess_repository(tmp_path) access = {record.function_name: record.access_level for record in analyze_entry_points(tmp_path, targets) if record.function_name} assert access == {"delete": "public", "get": "authenticated"} + + +def test_a_javascript_excerpt_never_starts_in_the_middle_of_an_import() -> None: + """`_declarator_start` walks back to the previous `;` or `}`. In JavaScript written without semicolons the + nearest `}` is the one in `import { expressify } from ...`, so the excerpt began mid-statement and the file + did not parse at all — a pair that cannot be parsed cannot be scored.""" + lib = _lib() + source = ( + "import { expressify } from '@overleaf/promise-utils'\n" + "import SessionManager from '../Authentication/SessionManager.mjs'\n\n" + "async function exportProject(req, res, next) {\n" + " return res.send(req.params.project_id)\n" + "}\n" + ) + excerpt = lib["extract_function"](source, "exportProject", "javascript") + assert excerpt.startswith("async function exportProject("), excerpt.splitlines()[0] + assert "from '@overleaf" not in excerpt + + +def test_the_overleaf_excerpt_parses_on_both_sides() -> None: + from openultrasast.semantic.ir import parse_file + + case = next( + item for item in select_vendored(load_pair_catalog(DEFAULT_CATALOG)) if item.name == "overleaf-with-claude-exportscontroller-3980b9" + ) + for path in (case.vuln_file, case.fixed_file): + ir = parse_file(case.relpath, path.read_text(), "javascript") + assert ir.parse_ok, f"{path.name}: {ir.reason}" + assert any(function.name == "exportProject" for function in ir.functions)