mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
fix(hunt): build output gets no candidate rows, a minified line is not scanned once per function, the screen checkpoints every change
The second access-control screen spun for 2h20m on one ordinary mage-ai change: a Next.js build commit whose 3 MB one-line bundles hold 12,355 functions each. `source.ranges_from_ir` ran two regex scans of the whole line once per function, and the screen kept its 422 finished changes in memory only, so stopping it lost them. - `source.ranges_from_ir`: a line over MINIFIED_LINE_CHARS (2000) is not checked per function; JavaScript names on it stay anonymous, other languages keep the parser's name. - `source.generated_reason`: why a changed file is build output (a directory the product walk leaves out, a minified name, over MAX_FILE_BYTES, a line over the cap). `hunt.producers._candidates` leaves such files out and records the reason on the context; the fix site of a vulnerability is always kept. The DSPy arm shares the enumeration. - The screen writes rows.jsonl and log.jsonl per change (rows first, entry last) and resumes by skipping logged ids. Tests: the minified line never reaches the per-function scan; generated_reason on the four shapes and two source files; build output in an ordinary change yields no rows while a vulnerability path in a one-line file is kept. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
4cf9c85d22
commit
81ae206978
@@ -55,6 +55,7 @@ class Context:
|
||||
texts: dict[str, str] = field(default_factory=dict)
|
||||
spans: dict[str, tuple[source.FunctionRange, ...]] = field(default_factory=dict)
|
||||
range_errors: dict[str, str] = field(default_factory=dict)
|
||||
excluded: dict[str, str] = field(default_factory=dict) # changed build output that got no candidate, by reason
|
||||
irs: dict[str, FileIR] = field(default_factory=dict)
|
||||
call_index: dict[str, tuple[source.CodeLine, ...]] = field(default_factory=dict)
|
||||
diff_bytes: int = 0
|
||||
@@ -324,6 +325,13 @@ def _candidates(root: Path, change: Change, ctx: Context) -> list[dict[str, Any]
|
||||
for file in files:
|
||||
path = file["path"]
|
||||
_safe_path(root, path)
|
||||
if not (change.get("kind") == "vulnerability" and path == change.get("path")):
|
||||
reason = source.generated_reason(root, path)
|
||||
if reason:
|
||||
# Build output carries no candidate: the product walk leaves it out, and parsing one 3 MB
|
||||
# minified bundle cost the screen two hours. The fix site of a vulnerability is always kept.
|
||||
ctx.excluded[path] = reason
|
||||
continue
|
||||
try:
|
||||
ranges = ctx.ranges(path)
|
||||
except Exception as error:
|
||||
@@ -363,7 +371,7 @@ def _candidates(root: Path, change: Change, ctx: Context) -> list[dict[str, Any]
|
||||
|
||||
|
||||
def candidate_rows(root: Path, change: Change) -> list[dict[str, Any]]:
|
||||
"""Enumerate touched head functions and the positive fix site, retaining unreadable files."""
|
||||
"""Enumerate touched head functions and the positive fix site, retaining unreadable files and leaving out build output."""
|
||||
root = Path(root)
|
||||
return _candidates(root, change, Context(root, change))
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ MODULE_FUNCTION = "<module>"
|
||||
LEGACY_MODULE_FUNCTION = "<global>" # what frozen records before 2026-10-13 call module-level code
|
||||
ANON_FUNCTION = "<anon>"
|
||||
MAX_FILE_BYTES = 1_000_000
|
||||
MINIFIED_LINE_CHARS = 2_000 # a longer line is minified output; scanning it once per function is quadratic
|
||||
LANGUAGES = frozenset({"python", "php", "javascript", "typescript", "java", "go", "ruby"}) # the declared languages
|
||||
_EXCLUDED_DIRS = frozenset(
|
||||
IGNORED_DIRS | {"vendor", "node_modules", "bower_components", "third_party", "third-party", "dist", "build", "out", "target"}
|
||||
@@ -238,6 +239,11 @@ def ranges_from_ir(ir: FileIR, text: str) -> tuple[FunctionRange, ...]:
|
||||
if fn.name == MODULE_FUNCTION:
|
||||
continue
|
||||
line = lines[fn.start_line - 1] if 0 < fn.start_line <= len(lines) else ""
|
||||
if len(line) > MINIFIED_LINE_CHARS:
|
||||
# A minified bundle puts thousands of functions on one line, and scanning that line once per function
|
||||
# took hours on a 3 MB bundle. Such a line is not checked: JavaScript names on it stay anonymous (the
|
||||
# parser names an arrow callback after its parameter), other languages keep the parser's name.
|
||||
line = ""
|
||||
declared = declarator_name(line) # the declaration tables would name a callback call (`exec(..., function () {`)
|
||||
name = declared or fn.name
|
||||
if (
|
||||
@@ -718,6 +724,32 @@ def product_paths(workspace: Path) -> list[Path]:
|
||||
return files
|
||||
|
||||
|
||||
_MINIFIED_NAME = re.compile(r"\.min\.(js|css)$|\.bundle\.js$")
|
||||
|
||||
|
||||
def generated_reason(root: Path, path: str) -> str | None:
|
||||
"""Why a changed file is build output rather than source, or None when it may be source.
|
||||
|
||||
The limits are the ones `product_files` applies: a directory the product walk leaves out, a minified file
|
||||
name, a file over MAX_FILE_BYTES, or a line over MINIFIED_LINE_CHARS. A file that cannot be read is not
|
||||
called generated; its reader reports that.
|
||||
"""
|
||||
relative = Path(path)
|
||||
if any(part in _EXCLUDED_DIRS for part in relative.parts[:-1]):
|
||||
return "excluded_dir"
|
||||
if _MINIFIED_NAME.search(relative.as_posix()):
|
||||
return "minified_name"
|
||||
try:
|
||||
if (root / relative).stat().st_size > MAX_FILE_BYTES:
|
||||
return "too_large"
|
||||
with (root / relative).open("rb") as stream:
|
||||
if any(len(line) > MINIFIED_LINE_CHARS for line in stream):
|
||||
return "long_line"
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def declared_functions(lines: Sequence[str], language: str) -> list[str]:
|
||||
"""The function names a file declares, in source order, each once."""
|
||||
pattern = DECLARATION.get(language)
|
||||
|
||||
Reference in New Issue
Block a user