mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
measure: the candidate generator must be the IR, not the ruleset
A candidate-driven detector can only judge sites something else enumerates,
so the enumerator caps recall. Measured over the 58 scorable web pairs before
building anything:
pattern ruleset: 12.1% ceiling -- and the misses are not near-misses, the
ruleset finds nothing at all in most of these files
semantic IR: 96.6% ceiling, a median of ten call sites and bindings
inside the labeled function
The ruleset would have capped us at 12.1% against the 64.9% the unconstrained
loop already reaches. That design was one commit from being built.
Two corpus defects found on the way, both mine from task 4.1. The overleaf
excerpt started mid-import, because `_declarator_start` walks back to the
nearest `}` and Overleaf writes JavaScript without semicolons, so the nearest
one was inside `import { expressify }`. It then ended mid-object-literal,
because registration mentions were appended as bare lines. Neither parsed,
and a pair that cannot be parsed cannot be scored. Registration mentions now
expand to the statement that contains them; the ceiling moved 94.8% -> 96.6%.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
4e6afa862b
commit
309faad266
@@ -425,3 +425,32 @@ def test_prose_inside_a_decorator_never_grants_a_guard(tmp_path: Path) -> None:
|
||||
_, targets = preprocess_repository(tmp_path)
|
||||
access = {record.function_name: record.access_level for record in analyze_entry_points(tmp_path, targets) if record.function_name}
|
||||
assert access == {"delete": "public", "get": "authenticated"}
|
||||
|
||||
|
||||
def test_a_javascript_excerpt_never_starts_in_the_middle_of_an_import() -> None:
|
||||
"""`_declarator_start` walks back to the previous `;` or `}`. In JavaScript written without semicolons the
|
||||
nearest `}` is the one in `import { expressify } from ...`, so the excerpt began mid-statement and the file
|
||||
did not parse at all — a pair that cannot be parsed cannot be scored."""
|
||||
lib = _lib()
|
||||
source = (
|
||||
"import { expressify } from '@overleaf/promise-utils'\n"
|
||||
"import SessionManager from '../Authentication/SessionManager.mjs'\n\n"
|
||||
"async function exportProject(req, res, next) {\n"
|
||||
" return res.send(req.params.project_id)\n"
|
||||
"}\n"
|
||||
)
|
||||
excerpt = lib["extract_function"](source, "exportProject", "javascript")
|
||||
assert excerpt.startswith("async function exportProject("), excerpt.splitlines()[0]
|
||||
assert "from '@overleaf" not in excerpt
|
||||
|
||||
|
||||
def test_the_overleaf_excerpt_parses_on_both_sides() -> None:
|
||||
from openultrasast.semantic.ir import parse_file
|
||||
|
||||
case = next(
|
||||
item for item in select_vendored(load_pair_catalog(DEFAULT_CATALOG)) if item.name == "overleaf-with-claude-exportscontroller-3980b9"
|
||||
)
|
||||
for path in (case.vuln_file, case.fixed_file):
|
||||
ir = parse_file(case.relpath, path.read_text(), "javascript")
|
||||
assert ir.parse_ok, f"{path.name}: {ir.reason}"
|
||||
assert any(function.name == "exportProject" for function in ir.functions)
|
||||
|
||||
Reference in New Issue
Block a user