From 81ae206978bad22434e69eda5df66c8ceccf5373 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:24:45 +0200 Subject: [PATCH] fix(hunt): build output gets no candidate rows, a minified line is not scanned once per function, the screen checkpoints every change The second access-control screen spun for 2h20m on one ordinary mage-ai change: a Next.js build commit whose 3 MB one-line bundles hold 12,355 functions each. `source.ranges_from_ir` ran two regex scans of the whole line once per function, and the screen kept its 422 finished changes in memory only, so stopping it lost them. - `source.ranges_from_ir`: a line over MINIFIED_LINE_CHARS (2000) is not checked per function; JavaScript names on it stay anonymous, other languages keep the parser's name. - `source.generated_reason`: why a changed file is build output (a directory the product walk leaves out, a minified name, over MAX_FILE_BYTES, a line over the cap). `hunt.producers._candidates` leaves such files out and records the reason on the context; the fix site of a vulnerability is always kept. The DSPy arm shares the enumeration. - The screen writes rows.jsonl and log.jsonl per change (rows first, entry last) and resumes by skipping logged ids. Tests: the minified line never reaches the per-function scan; generated_reason on the four shapes and two source files; build output in an ordinary change yields no rows while a vulnerability path in a one-line file is kept. Co-Authored-By: Claude Fable 5.1 --- .../2026-10-13-access-control-screen/table.py | 53 ++++++++++++++----- src/openultrasast/hunt/producers.py | 10 +++- src/openultrasast/source.py | 32 +++++++++++ tests/test_hunt_producers.py | 15 ++++++ tests/test_source.py | 27 ++++++++++ 5 files changed, 124 insertions(+), 13 deletions(-) diff --git a/benchmarks/measurements/2026-10-13-access-control-screen/table.py b/benchmarks/measurements/2026-10-13-access-control-screen/table.py index 68536b9..93e95c6 100644 --- a/benchmarks/measurements/2026-10-13-access-control-screen/table.py +++ b/benchmarks/measurements/2026-10-13-access-control-screen/table.py @@ -25,7 +25,7 @@ from pathlib import Path from openultrasast.hunt import features from openultrasast.hunt.producers import compute_rows from openultrasast.hunt.runner import _full_checkout as full_checkout -from openultrasast.hunt.table import FeatureTable +from openultrasast.hunt.table import FeatureTable, canonical_json from openultrasast.stats import separation @@ -84,6 +84,20 @@ def unread_rows(rows: list[dict], repos: dict[str, str]) -> int: return count +def resume(rows_path: Path, log_path: Path) -> tuple[list[dict], list[dict]]: + """Rows and log entries an earlier run wrote into the same output directory, one change at a time. + + The first screen over the second pool kept every row in memory and lost 422 finished changes when one change + had to be stopped; now each change lands on disk as it finishes and a restart skips the finished ones. + """ + if not log_path.exists(): + return [], [] + log = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + done = {entry["id"] for entry in log} + rows = [json.loads(line) for line in rows_path.read_text(encoding="utf-8").splitlines() if line.strip()] if rows_path.exists() else [] + return [row for row in rows if row["change_id"] in done], log + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("pool", type=Path) @@ -93,17 +107,32 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) repos = json.loads(args.repos.read_text()) args.out.mkdir(parents=True, exist_ok=True) - rows, log = [], [] - for change in changes_of(args.pool, args.slice): - clone = Path(repos[change["repo"]]).expanduser() - try: - produced, info = rows_for(change, clone) - except Exception as error: # noqa: BLE001 (type only; a message can carry a path) - info = {"failure": type(error).__name__} - produced = [] - rows.extend(produced) - log.append({"id": change["id"], "repo": change["repo"], "kind": change["kind"], **info}) - print(f"{change['id'][:16]} {change['kind']} {info}", flush=True) + rows_path, log_path = args.out / "rows.jsonl", args.out / "log.jsonl" + rows, log = resume(rows_path, log_path) + done = {entry["id"] for entry in log} + if done: + print(f"resuming: {len(done)} changes and {len(rows)} rows already on disk", flush=True) + with rows_path.open("a", encoding="utf-8") as rows_stream, log_path.open("a", encoding="utf-8") as log_stream: + for change in changes_of(args.pool, args.slice): + if change["id"] in done: + continue + clone = Path(repos[change["repo"]]).expanduser() + try: + produced, info = rows_for(change, clone) + except Exception as error: # noqa: BLE001 (type only; a message can carry a path) + info = {"failure": type(error).__name__} + produced = [] + rows.extend(produced) + entry = {"id": change["id"], "repo": change["repo"], "kind": change["kind"], **info} + log.append(entry) + # Rows first, the log entry last: a change without its entry is recomputed on resume and the rows it + # wrote are dropped, so a kill between the two writes never duplicates rows. + for row in produced: + rows_stream.write(canonical_json(row) + "\n") + rows_stream.flush() + log_stream.write(json.dumps(entry) + "\n") + log_stream.flush() + print(f"{change['id'][:16]} {change['kind']} {info}", flush=True) table = FeatureTable(rows) table.write_jsonl(args.out / "table.jsonl") # The instrument gate: a producer that failed on every row saw an unread tree, not a feature-free one. diff --git a/src/openultrasast/hunt/producers.py b/src/openultrasast/hunt/producers.py index 4053c2d..653d985 100644 --- a/src/openultrasast/hunt/producers.py +++ b/src/openultrasast/hunt/producers.py @@ -55,6 +55,7 @@ class Context: texts: dict[str, str] = field(default_factory=dict) spans: dict[str, tuple[source.FunctionRange, ...]] = field(default_factory=dict) range_errors: dict[str, str] = field(default_factory=dict) + excluded: dict[str, str] = field(default_factory=dict) # changed build output that got no candidate, by reason irs: dict[str, FileIR] = field(default_factory=dict) call_index: dict[str, tuple[source.CodeLine, ...]] = field(default_factory=dict) diff_bytes: int = 0 @@ -324,6 +325,13 @@ def _candidates(root: Path, change: Change, ctx: Context) -> list[dict[str, Any] for file in files: path = file["path"] _safe_path(root, path) + if not (change.get("kind") == "vulnerability" and path == change.get("path")): + reason = source.generated_reason(root, path) + if reason: + # Build output carries no candidate: the product walk leaves it out, and parsing one 3 MB + # minified bundle cost the screen two hours. The fix site of a vulnerability is always kept. + ctx.excluded[path] = reason + continue try: ranges = ctx.ranges(path) except Exception as error: @@ -363,7 +371,7 @@ def _candidates(root: Path, change: Change, ctx: Context) -> list[dict[str, Any] def candidate_rows(root: Path, change: Change) -> list[dict[str, Any]]: - """Enumerate touched head functions and the positive fix site, retaining unreadable files.""" + """Enumerate touched head functions and the positive fix site, retaining unreadable files and leaving out build output.""" root = Path(root) return _candidates(root, change, Context(root, change)) diff --git a/src/openultrasast/source.py b/src/openultrasast/source.py index c830d0b..46d0c9a 100644 --- a/src/openultrasast/source.py +++ b/src/openultrasast/source.py @@ -43,6 +43,7 @@ MODULE_FUNCTION = "" LEGACY_MODULE_FUNCTION = "" # what frozen records before 2026-10-13 call module-level code ANON_FUNCTION = "" MAX_FILE_BYTES = 1_000_000 +MINIFIED_LINE_CHARS = 2_000 # a longer line is minified output; scanning it once per function is quadratic LANGUAGES = frozenset({"python", "php", "javascript", "typescript", "java", "go", "ruby"}) # the declared languages _EXCLUDED_DIRS = frozenset( IGNORED_DIRS | {"vendor", "node_modules", "bower_components", "third_party", "third-party", "dist", "build", "out", "target"} @@ -238,6 +239,11 @@ def ranges_from_ir(ir: FileIR, text: str) -> tuple[FunctionRange, ...]: if fn.name == MODULE_FUNCTION: continue line = lines[fn.start_line - 1] if 0 < fn.start_line <= len(lines) else "" + if len(line) > MINIFIED_LINE_CHARS: + # A minified bundle puts thousands of functions on one line, and scanning that line once per function + # took hours on a 3 MB bundle. Such a line is not checked: JavaScript names on it stay anonymous (the + # parser names an arrow callback after its parameter), other languages keep the parser's name. + line = "" declared = declarator_name(line) # the declaration tables would name a callback call (`exec(..., function () {`) name = declared or fn.name if ( @@ -718,6 +724,32 @@ def product_paths(workspace: Path) -> list[Path]: return files +_MINIFIED_NAME = re.compile(r"\.min\.(js|css)$|\.bundle\.js$") + + +def generated_reason(root: Path, path: str) -> str | None: + """Why a changed file is build output rather than source, or None when it may be source. + + The limits are the ones `product_files` applies: a directory the product walk leaves out, a minified file + name, a file over MAX_FILE_BYTES, or a line over MINIFIED_LINE_CHARS. A file that cannot be read is not + called generated; its reader reports that. + """ + relative = Path(path) + if any(part in _EXCLUDED_DIRS for part in relative.parts[:-1]): + return "excluded_dir" + if _MINIFIED_NAME.search(relative.as_posix()): + return "minified_name" + try: + if (root / relative).stat().st_size > MAX_FILE_BYTES: + return "too_large" + with (root / relative).open("rb") as stream: + if any(len(line) > MINIFIED_LINE_CHARS for line in stream): + return "long_line" + except OSError: + return None + return None + + def declared_functions(lines: Sequence[str], language: str) -> list[str]: """The function names a file declares, in source order, each once.""" pattern = DECLARATION.get(language) diff --git a/tests/test_hunt_producers.py b/tests/test_hunt_producers.py index 8168668..00d8041 100644 --- a/tests/test_hunt_producers.py +++ b/tests/test_hunt_producers.py @@ -256,3 +256,18 @@ def test_obligation_loader_errors_remain_type_only(repository, monkeypatch): assert row["provenance.semantic.ok"] is False assert row["provenance.semantic.reason"] == "OSError" assert "private file contents" not in json.dumps(row) + + +def test_build_output_in_an_ordinary_change_gets_no_candidate_rows(repository): + root, change = repository + (root / "dist").mkdir() + (root / "dist" / "bundle.js").write_text("function a() {}\n") + (root / "app.min.js").write_text("function b() {}\n") + (root / "one.js").write_text("".join(f"function f{i}(){{}}" for i in range(400)) + "\n") + (root / "big.js").write_text("function c() {}\n" + "// padding\n" * 100_000) + files = [{"path": p, "head_lines": [[1, 1]]} for p in ("dist/bundle.js", "app.min.js", "one.js", "big.js", "app.py")] + rows = candidate_rows(root, {**change, "kind": "ordinary", "files": files}) + assert {row["path"] for row in rows} == {"app.py"} + # The fix site of a vulnerability is kept whatever the file looks like. + kept = candidate_rows(root, {**change, "path": "one.js", "function": "f0", "function_lines": [1, 1], "files": files[2:3]}) + assert [row["path"] for row in kept] == ["one.js"] diff --git a/tests/test_source.py b/tests/test_source.py index 374c768..f3772f7 100644 --- a/tests/test_source.py +++ b/tests/test_source.py @@ -162,3 +162,30 @@ def test_call_index_gives_the_same_call_sites_and_is_reused_across_names(tmp_pat assert source.callers_of(tmp_path, "handler", files=["other.py"], index=cache) == (("other.py", 6),) (tmp_path / "other.py").write_text("") # the cached index, not the disk, answers the second name assert source.callers_of(tmp_path, "other", files=["other.py"], index=cache) == (("other.py", 7),) + + +def test_a_minified_line_is_not_scanned_once_per_function(monkeypatch): + text = "".join(f"function f{i}(){{return {i}}}" for i in range(3000)) + "\n" + assert len(text) > source.MINIFIED_LINE_CHARS + calls: list[str] = [] + original = source._declares + monkeypatch.setattr(source, "_declares", lambda line, name: calls.append(line) or original(line, name)) + result = source.ranges("bundle.js", text, "javascript") + assert len(result) == 3001 and result[0].name == source.MODULE_FUNCTION + assert set(calls) == {""} # the long line itself never reaches the per-function scan + assert {r.name for r in result[1:]} == {source.ANON_FUNCTION} + + +def test_generated_reason_names_build_output_only(tmp_path): + (tmp_path / "dist").mkdir() + (tmp_path / "dist" / "bundle.js").write_text("function a() {}\n") + (tmp_path / "app.min.js").write_text("function b() {}\n") + (tmp_path / "one.js").write_text("".join(f"function f{i}(){{}}" for i in range(400)) + "\n") + (tmp_path / "big.js").write_text("function c() {}\n" + "// padding\n" * 100_000) + (tmp_path / "app.js").write_text("function d() {}\n") + assert source.generated_reason(tmp_path, "dist/bundle.js") == "excluded_dir" + assert source.generated_reason(tmp_path, "app.min.js") == "minified_name" + assert source.generated_reason(tmp_path, "one.js") == "long_line" + assert source.generated_reason(tmp_path, "big.js") == "too_large" + assert source.generated_reason(tmp_path, "app.js") is None + assert source.generated_reason(tmp_path, "missing.js") is None