fix(hunt): build output gets no candidate rows, a minified line is not scanned once per function, the screen checkpoints every change

The second access-control screen spun for 2h20m on one ordinary mage-ai change: a Next.js build commit whose 3 MB
one-line bundles hold 12,355 functions each. `source.ranges_from_ir` ran two regex scans of the whole line once per
function, and the screen kept its 422 finished changes in memory only, so stopping it lost them.

- `source.ranges_from_ir`: a line over MINIFIED_LINE_CHARS (2000) is not checked per function; JavaScript names on it
  stay anonymous, other languages keep the parser's name.
- `source.generated_reason`: why a changed file is build output (a directory the product walk leaves out, a minified
  name, over MAX_FILE_BYTES, a line over the cap). `hunt.producers._candidates` leaves such files out and records the
  reason on the context; the fix site of a vulnerability is always kept. The DSPy arm shares the enumeration.
- The screen writes rows.jsonl and log.jsonl per change (rows first, entry last) and resumes by skipping logged ids.

Tests: the minified line never reaches the per-function scan; generated_reason on the four shapes and two source files;
build output in an ordinary change yields no rows while a vulnerability path in a one-line file is kept.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-14 13:24:45 +02:00
co-authored by Claude Fable 5.1
parent 4cf9c85d22
commit 81ae206978
5 changed files with 124 additions and 13 deletions
+9 -1
View File
@@ -55,6 +55,7 @@ class Context:
texts: dict[str, str] = field(default_factory=dict)
spans: dict[str, tuple[source.FunctionRange, ...]] = field(default_factory=dict)
range_errors: dict[str, str] = field(default_factory=dict)
excluded: dict[str, str] = field(default_factory=dict) # changed build output that got no candidate, by reason
irs: dict[str, FileIR] = field(default_factory=dict)
call_index: dict[str, tuple[source.CodeLine, ...]] = field(default_factory=dict)
diff_bytes: int = 0
@@ -324,6 +325,13 @@ def _candidates(root: Path, change: Change, ctx: Context) -> list[dict[str, Any]
for file in files:
path = file["path"]
_safe_path(root, path)
if not (change.get("kind") == "vulnerability" and path == change.get("path")):
reason = source.generated_reason(root, path)
if reason:
# Build output carries no candidate: the product walk leaves it out, and parsing one 3 MB
# minified bundle cost the screen two hours. The fix site of a vulnerability is always kept.
ctx.excluded[path] = reason
continue
try:
ranges = ctx.ranges(path)
except Exception as error:
@@ -363,7 +371,7 @@ def _candidates(root: Path, change: Change, ctx: Context) -> list[dict[str, Any]
def candidate_rows(root: Path, change: Change) -> list[dict[str, Any]]:
"""Enumerate touched head functions and the positive fix site, retaining unreadable files."""
"""Enumerate touched head functions and the positive fix site, retaining unreadable files and leaving out build output."""
root = Path(root)
return _candidates(root, change, Context(root, change))
+32
View File
@@ -43,6 +43,7 @@ MODULE_FUNCTION = "<module>"
LEGACY_MODULE_FUNCTION = "<global>" # what frozen records before 2026-10-13 call module-level code
ANON_FUNCTION = "<anon>"
MAX_FILE_BYTES = 1_000_000
MINIFIED_LINE_CHARS = 2_000 # a longer line is minified output; scanning it once per function is quadratic
LANGUAGES = frozenset({"python", "php", "javascript", "typescript", "java", "go", "ruby"}) # the declared languages
_EXCLUDED_DIRS = frozenset(
IGNORED_DIRS | {"vendor", "node_modules", "bower_components", "third_party", "third-party", "dist", "build", "out", "target"}
@@ -238,6 +239,11 @@ def ranges_from_ir(ir: FileIR, text: str) -> tuple[FunctionRange, ...]:
if fn.name == MODULE_FUNCTION:
continue
line = lines[fn.start_line - 1] if 0 < fn.start_line <= len(lines) else ""
if len(line) > MINIFIED_LINE_CHARS:
# A minified bundle puts thousands of functions on one line, and scanning that line once per function
# took hours on a 3 MB bundle. Such a line is not checked: JavaScript names on it stay anonymous (the
# parser names an arrow callback after its parameter), other languages keep the parser's name.
line = ""
declared = declarator_name(line) # the declaration tables would name a callback call (`exec(..., function () {`)
name = declared or fn.name
if (
@@ -718,6 +724,32 @@ def product_paths(workspace: Path) -> list[Path]:
return files
_MINIFIED_NAME = re.compile(r"\.min\.(js|css)$|\.bundle\.js$")
def generated_reason(root: Path, path: str) -> str | None:
"""Why a changed file is build output rather than source, or None when it may be source.
The limits are the ones `product_files` applies: a directory the product walk leaves out, a minified file
name, a file over MAX_FILE_BYTES, or a line over MINIFIED_LINE_CHARS. A file that cannot be read is not
called generated; its reader reports that.
"""
relative = Path(path)
if any(part in _EXCLUDED_DIRS for part in relative.parts[:-1]):
return "excluded_dir"
if _MINIFIED_NAME.search(relative.as_posix()):
return "minified_name"
try:
if (root / relative).stat().st_size > MAX_FILE_BYTES:
return "too_large"
with (root / relative).open("rb") as stream:
if any(len(line) > MINIFIED_LINE_CHARS for line in stream):
return "long_line"
except OSError:
return None
return None
def declared_functions(lines: Sequence[str], language: str) -> list[str]:
"""The function names a file declares, in source order, each once."""
pattern = DECLARATION.get(language)