feat(pre-push-safety-net): exclude vendor inputs and partition frontend coverage

This commit is contained in:
Marius Ciepluch
2026-09-13 11:53:29 +02:00
parent 51fabef9ea
commit 6ba2b06353
29 changed files with 1996 additions and 72 deletions
+2 -1
View File
@@ -4,7 +4,7 @@ import argparse
import json
import tempfile
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, replace
from dataclasses import asdict, dataclass, replace
from pathlib import Path
from time import perf_counter
from typing import TYPE_CHECKING, Any
@@ -903,6 +903,7 @@ def _model_payload(
"scope": scope.to_payload() if scope is not None else None,
"question_outcomes": [q.to_payload() for q in outcomes],
"family_coverage": [c.to_payload() for c in getattr(result, "family_coverage", ())],
"partitions": [asdict(p) for p in getattr(result, "partitions", ())],
"by_rung": dict(getattr(result, "by_rung", {}) or {}),
"regions_scanned": getattr(result, "regions_scanned", 0),
"regions_unjudged": getattr(result, "regions_unjudged", 0),
+14 -1
View File
@@ -697,9 +697,12 @@ class JoernBackend:
methods = 0
files = 0
answered = 0
file_names: set[str] = set()
named_census = True
for shard in shards:
one = self.query_batch(shard, query, requests)
if one is None:
named_census = False
if self.execution_budget is not None:
self._note("shard_query_incomplete")
return None
@@ -709,11 +712,21 @@ class JoernBackend:
if census and isinstance(census[0], Mapping):
methods += int(str(census[0].get("methods", "0")) or 0)
files += int(str(census[0].get("files", "0")) or 0)
names = census[0].get("file_names")
if isinstance(names, list) and all(isinstance(name, str) for name in names):
file_names.update(names)
else:
named_census = False
else:
named_census = False
for rid, rows in one.items():
merged.setdefault(rid, []).extend(rows)
if not answered:
return None
merged["__census__"] = [{"methods": str(methods), "files": str(files), "shards": str(len(shards))}]
total: dict[str, object] = {"methods": str(methods), "files": str(files), "shards": str(len(shards))}
if named_census:
total["file_names"] = sorted(file_names)
merged["__census__"] = [total]
return merged
def _apply_overlays(self, cpg_path: Path, scratch: Path) -> bool:
+1 -1
View File
@@ -78,7 +78,7 @@
// per-file warnings -- so "no rows" and "no graph" are indistinguishable to the driver without this.
// It rides the batch rather than costing its own invocation, because JVM startup is what this whole
// batching design exists to avoid.
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString))
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString, "file_names" -> ujson.Arr.from(cpg.file.name.l)))
println(ujson.write(ujson.Obj.from(answers.toSeq :+ ("__census__" -> census))))
} else {
println(ujson.write(ujson.Arr(rowsFor(settings, function, file): _*)))
+1 -1
View File
@@ -190,7 +190,7 @@
// per-file warnings -- so "no rows" and "no graph" are indistinguishable to the driver without this.
// It rides the batch rather than costing its own invocation, because JVM startup is what this whole
// batching design exists to avoid.
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString))
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString, "file_names" -> ujson.Arr.from(cpg.file.name.l)))
println(ujson.write(ujson.Obj.from(answers.toSeq :+ ("__census__" -> census))))
} else {
println(ujson.write(ujson.Arr(rowsFor(operations, dischargers, function, file, operationRequires, dischargersByKind): _*)))
+1 -1
View File
@@ -852,7 +852,7 @@
// per-file warnings -- so "no rows" and "no graph" are indistinguishable to the driver without this.
// It rides the batch rather than costing its own invocation, because JVM startup is what this whole
// batching design exists to avoid.
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString))
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString, "file_names" -> ujson.Arr.from(cpg.file.name.l)))
println(ujson.write(ujson.Obj.from(answers.toSeq :+ ("__census__" -> census))))
} else {
println(
+6 -1
View File
@@ -26,7 +26,12 @@ def _matches(relative: str, patterns: Iterable[str]) -> bool:
parts = relative.split("/")
for pattern in patterns:
if pattern.endswith("/"):
if pattern.rstrip("/") in parts[:-1] or pattern.rstrip("/") in parts:
directory = pattern.rstrip("/")
if any(
fnmatch.fnmatch("/".join(parts[start:end]), directory)
for start in range(len(parts))
for end in range(start + 1, len(parts) + 1)
):
return True
elif fnmatch.fnmatch(relative, pattern) or fnmatch.fnmatch(parts[-1], pattern):
return True
+279
View File
@@ -0,0 +1,279 @@
"""Physical first-party frontend inputs and routing; selection remains in the scan ranker."""
from __future__ import annotations
import os
import stat
import tempfile
import time
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from ..cpg.backend import _DeadlineExpired, _remove_owned_tree
from ..preprocess import LANGUAGE_BY_EXTENSION
from .contracts import ExecutionBudget
from .layout import is_test_path, is_vendored, layout_facts
FRONTENDS = {
"php": "php",
"javascript": "javascript",
"typescript": "typescript",
"python": "python",
"java": "java",
"c": "c",
"cpp": "c_cpp",
}
@dataclass(frozen=True)
class PartitionCoverage:
language: str
frontend: str | None
paths: tuple[str, ...]
source_bytes: int
status: str
boundaries: tuple[str, ...] = ()
runtime: str = "unspecified"
excluded_paths: tuple[str, ...] = ()
unshipped_paths: tuple[str, ...] = ()
class PartitionGraph:
"""Routes existing requests without reordering or allocating another scan budget."""
def __init__(self, budget: ExecutionBudget | None = None) -> None:
self.budget = budget
self.diagnostics: list[str] = []
self.graphs: dict[str, Any] = {}
self.roots: dict[str, Path] = {}
self.path_languages: dict[str, str] = {}
self.partitions: tuple[PartitionCoverage, ...] = ()
self.boundaries: tuple[str, ...] = ()
self.unparsed: tuple[str, ...] = ()
self.cpg_path = Path("partitioned-cpg")
def cleanup(self) -> None:
try:
for graph in self.graphs.values():
cleanup = getattr(graph, "cleanup", None)
if callable(cleanup):
cleanup()
finally:
for root in self.roots.values():
deadline = self.budget.deadline_monotonic + self.budget.cancellation_allowance_seconds if self.budget else float("inf")
try:
_remove_owned_tree(root, deadline)
except (OSError, _DeadlineExpired):
self.diagnostics.append("scratch_cleanup_incomplete")
def execution_diagnostics(self) -> tuple[str, ...]:
reasons = list(self.diagnostics)
for graph in self.graphs.values():
read = getattr(graph, "execution_diagnostics", None)
if callable(read):
reasons.extend(read())
return tuple(reasons)
def _normalize(self, value: Any) -> Any:
if isinstance(value, str):
for root in self.roots.values():
value = value.replace(str(root) + "/", "")
return value
if isinstance(value, list):
return [self._normalize(v) for v in value]
if isinstance(value, dict):
return {k: self._normalize(v) for k, v in value.items()}
return value
def run(self, kind: str, params: Mapping[str, object]) -> list[object] | None:
return self.run_batch(kind, {"one": params}).get("one")
def run_batch(self, kind: str, requests: Mapping[str, Mapping[str, object]]) -> dict[str, list[object]]:
answers: dict[str, list[object]] = {}
languages = dict.fromkeys(self.path_languages.get(str(params.get("file", "")), "") for params in requests.values())
for language in languages:
graph = self.graphs.get(language)
if graph is None:
continue
if self.budget is not None and time.monotonic() >= self.budget.deadline_monotonic:
break
selected = {rid: params for rid, params in requests.items() if self.path_languages.get(str(params.get("file", ""))) == language}
if not selected:
continue
batch = getattr(graph, "run_batch", None)
raw = batch(kind, selected) if callable(batch) else {rid: graph.run(kind, params) for rid, params in selected.items()}
if not isinstance(raw, Mapping):
continue
for rid, rows in raw.items():
if rid in selected and isinstance(rows, list):
answers[rid] = self._normalize(rows)
elif rid == "__census__" and isinstance(rows, list):
expected = set(next(p.paths for p in self.partitions if p.language == language))
normalized = self._normalize(rows)
names = [r.get("file_names") for r in normalized if isinstance(r, Mapping)]
named = bool(names) and all(isinstance(n, list) and all(isinstance(p, str) for p in n) for n in names)
observed = {p.removeprefix("./") for n in names if isinstance(n, list) for p in n if isinstance(p, str)}
if not named or not expected <= observed:
self.diagnostics.append("partition_file_census_incomplete" if named else "partition_file_census_unavailable")
rows = [
{
"methods": 0,
"files": 0,
"shards": max((int(str(r.get("shards", 1))) for r in rows if isinstance(r, Mapping)), default=1),
}
]
# Preserve any individual empty or sharded graph's degradation.
previous = answers.get(rid, [])
if not previous or any(
isinstance(r, Mapping) and (int(str(r.get("methods", 0))) == 0 or int(str(r.get("shards", 1))) > 1) for r in rows
):
answers[rid] = rows
return answers
def build_partitions(
root: Path, build: Callable[[Path, str], Any], budget: ExecutionBudget | None, declared: Mapping[str, tuple[str, ...]] | None = None
) -> PartitionGraph:
graph = PartitionGraph(budget)
paths: dict[str, list[Path]] = {}
boundaries: set[str] = set()
excluded: set[str] = set()
facts = layout_facts()
root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
for directory, dirs, files, directory_fd in os.fwalk(
".", follow_symlinks=False, dir_fd=root_fd, onerror=lambda error: boundaries.add("source_unreadable")
):
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
boundaries.add("deadline_exhausted")
break
base = root / directory
for name in list(dirs):
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
boundaries.add("deadline_exhausted")
dirs.clear()
break
path = base / name
relative = path.relative_to(root).as_posix()
if name in {".git", ".hg", ".svn", ".openultrasast", ".venv", "venv", "__pycache__"}:
excluded.add(relative + "/")
dirs.remove(name)
continue
if path.is_symlink() or is_vendored(relative + "/", facts):
excluded.add(relative + "/")
boundaries.add("symlink_context_unresolved" if path.is_symlink() else "vendor_semantics_unresolved")
dirs.remove(name)
for name in files:
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
boundaries.add("deadline_exhausted")
break
path = base / name
relative = path.relative_to(root).as_posix()
if is_vendored(relative, facts):
excluded.add(relative)
boundaries.add("vendor_semantics_unresolved")
continue
if path.is_symlink() or not path.is_file():
excluded.add(relative)
boundaries.add("symlink_context_unresolved")
continue
language = LANGUAGE_BY_EXTENSION.get(path.suffix.lower(), "unknown")
if language == "unknown":
try:
fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory_fd)
with os.fdopen(fd, "rb") as handle:
first = handle.readline(4096).lower()
except OSError:
boundaries.add("source_unreadable")
continue
if first.startswith(b"#!"):
language = "python" if b"python" in first else "javascript" if b"node" in first else "unknown"
languages = (language,) if language != "unknown" else (declared or {}).get(relative, ())
for item in languages:
paths.setdefault(item, []).append(path)
if len(languages) == 1:
graph.path_languages[relative] = languages[0]
elif languages:
boundaries.add("ambiguous_frontend_path")
if len(paths) > 1:
boundaries.add("cross_partition_semantics_unresolved")
records: list[PartitionCoverage] = []
for language, sources in sorted(paths.items()):
relative_paths = tuple(sorted(p.relative_to(root).as_posix() for p in sources))
frontend = FRONTENDS.get(language)
if frontend is None:
records.append(PartitionCoverage(language, None, relative_paths, 0, "unsupported", ("frontend_unsupported",)))
boundaries.add("frontend_unsupported")
continue
scratch = Path(tempfile.mkdtemp(prefix="ousast-input-"))
graph.roots[language] = scratch
size = 0
complete = True
for source in sources:
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
complete = False
break
target = scratch / source.relative_to(root)
target.parent.mkdir(parents=True, exist_ok=True)
# A no-follow open prevents file symlink substitution between census and read.
try:
fd = _open_source(root_fd, source.relative_to(root).parts)
except OSError:
boundaries.add("source_unreadable")
complete = False
break
with os.fdopen(fd, "rb") as handle, target.open("wb") as output:
while chunk := handle.read(1024 * 1024):
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
complete = False
break
output.write(chunk)
size += len(chunk)
if not complete:
break
built = build(scratch, frontend) if complete else None
if built is not None:
graph.graphs[language] = built
graph.unparsed += tuple(graph._normalize(list(getattr(built, "unparsed", ()))))
record_limits: tuple[str, ...] = ("runtime_unspecified",) if language in {"javascript", "typescript"} else ()
if language == "typescript":
record_limits += ("typescript_property_support_unvalidated",)
boundaries.add("typescript_property_support_unvalidated")
if language in {"c", "cpp"}:
record_limits += ("c_bounds_arithmetic_unmodeled",)
records.append(
PartitionCoverage(language, frontend, relative_paths, size, "built" if built is not None else "build_failed", record_limits)
)
from dataclasses import replace
graph.partitions = tuple(
replace(
record, excluded_paths=tuple(sorted(excluded)), unshipped_paths=tuple(p for p in record.paths if is_test_path(p, facts))
)
for record in records
)
graph.boundaries = tuple(sorted(boundaries))
return graph
except BaseException:
graph.cleanup()
raise
finally:
os.close(root_fd)
def _open_source(root_fd: int, parts: tuple[str, ...]) -> int:
descriptor = os.dup(root_fd)
try:
for component in parts[:-1]:
child = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor)
os.close(descriptor)
descriptor = child
result = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=descriptor)
if not stat.S_ISREG(os.fstat(result).st_mode):
os.close(result)
raise OSError("source is not a regular file")
return result
finally:
os.close(descriptor)
+60 -8
View File
@@ -43,7 +43,8 @@ from .contracts import (
from .dominance import request_params as dominance_params
from .evidence import TIER_EXCLUDE, Evidence, evidence_from_rows
from .ladder import Rung
from .layout import with_layout
from .layout import is_vendored, layout_facts, with_layout
from .partitions import PartitionCoverage, build_partitions
from .pipeline import ModelFinding, scan_region
from .regions import MODULE_SCOPE, ScanRegion, affected_context
from .specs import ConfigSpec, DominanceSpec, TaintSpec, config_specs, dominance_specs, taint_specs
@@ -126,6 +127,7 @@ class ModelScanResult:
# could have been found was skipped. Reported so the request count is legible.
requests_pruned: int = 0
partitions: tuple[PartitionCoverage, ...] = ()
change_context: ChangeContext | None = None
scope: ScopeDecision | None = None
question_outcomes: tuple[QuestionOutcome, ...] = ()
@@ -158,6 +160,8 @@ def scan_repository(
if ranking_mode is not None:
limits = replace(limits, order_by_evidence=ranking_mode == "evidence")
mode = "evidence" if limits.order_by_evidence else "static"
layout = layout_facts()
regions = tuple(r for r in regions if not is_vendored(r.path, layout))
# Reject duplicate canonical questions before any engine work or resource ownership.
identities = [_identity(unit, r, f) for r in regions for f in r.families]
if len(set(identities)) != len(identities):
@@ -223,13 +227,29 @@ def _scan_repository_impl(
for region in regions:
counts[region.language] = counts.get(region.language, 0) + 1
dominant = max(counts, key=lambda name: (counts[name], name)) if counts else ""
cpg = _build(backend, root, dominant, execution_budget=execution_budget)
declarations: dict[str, set[str]] = {}
for region in regions:
declarations.setdefault(region.path, set()).add(region.language)
if root.is_dir():
cpg = build_partitions(
root,
lambda source, language: _build(backend, source, language, execution_budget=execution_budget),
execution_budget,
{path: tuple(sorted(languages)) for path, languages in declarations.items()},
)
degradations.extend({"stage": "model", "reason": reason} for reason in cpg.boundaries)
else:
cpg = _build(backend, root, dominant, execution_budget=execution_budget)
build_seconds = round(time.monotonic() - build_started, 2)
if cpg is None:
if cpg is None or (hasattr(cpg, "graphs") and not cpg.graphs):
if cpg is not None:
owned.append(cpg)
scope, _ = _scope_work(
regions, {}, limits, unit, ranking_mode, population_complete, failure="cpg_build_failed", change_context=change_context
)
scope = replace(scope, unresolved_boundaries=tuple(dict.fromkeys((*scope.unresolved_boundaries, *getattr(cpg, "boundaries", ())))))
return ModelScanResult(
partitions=getattr(cpg, "partitions", ()),
change_context=change_context,
scope=scope,
family_coverage=_family_coverage(scope, ()),
@@ -322,7 +342,22 @@ def _scan_repository_impl(
# Stable question IDs are the IDs sent to every execution request below.
scope_limits = (
replace(limits, prune_tier0=False)
if any(d.get("reason") in {"files_unparsed", "cpg_empty", "cpg_sharded"} for d in degradations)
if any(
d.get("reason")
in {
"files_unparsed",
"cpg_empty",
"cpg_sharded",
"cross_partition_semantics_unresolved",
"vendor_semantics_unresolved",
"symlink_context_unresolved",
"frontend_unsupported",
"source_unreadable",
"ambiguous_frontend_path",
"typescript_property_support_unvalidated",
}
for d in degradations
)
else limits
)
scope, work = _scope_work(
@@ -457,16 +492,32 @@ def _scan_repository_impl(
if unasked:
degradations.append({"stage": "model", "reason": "budget_exhausted", "regions_unasked": len(unasked)})
graph_gaps = tuple(str(d["reason"]) for d in degradations if d.get("reason") in {"files_unparsed", "cpg_empty", "cpg_sharded"})
graph_gaps = tuple(
str(d["reason"])
for d in degradations
if d.get("reason")
in {
"files_unparsed",
"cpg_empty",
"cpg_sharded",
"cross_partition_semantics_unresolved",
"vendor_semantics_unresolved",
"symlink_context_unresolved",
"frontend_unsupported",
"source_unreadable",
"ambiguous_frontend_path",
"typescript_property_support_unvalidated",
}
)
for question in scope.selected:
rid = question.identity.question_id
if rid not in outcomes:
answered = rid in rows_by_id
has_answer = rid in rows_by_id
outcomes[rid] = QuestionOutcome(
question.identity,
"not_arbitrated" if answered else "unanswered",
"not_arbitrated" if has_answer else "unanswered",
"deadline_exhausted" if not _within_deadline(execution_budget) else "query_unanswered",
_rows_json(rows_by_id[rid]) if answered else None,
_rows_json(rows_by_id[rid]) if has_answer else None,
)
if question.identity in context_gaps and outcomes[rid].status == "completed":
outcomes[rid] = replace(outcomes[rid], status="unresolved", reason="change_context_incomplete")
@@ -480,6 +531,7 @@ def _scan_repository_impl(
# The graph has answered everything it is going to. Its scratch tree holds the CPG, the request files and
# joern's own working copy -- tens of megabytes per scan -- and nothing else reclaims it.
return ModelScanResult(
partitions=getattr(cpg, "partitions", ()),
findings=findings,
by_rung=_tally(findings),
regions_scanned=scanned,
+7 -1
View File
@@ -97,7 +97,13 @@ def enumerate_source_files(root: Path) -> list[Path]:
patterns = _load_ignore_patterns(root)
paths: list[Path] = []
for path in root.rglob("*"):
if not path.is_file() or _has_ignored_dir(root, path) or _is_ignored(root, path, patterns) or _is_vendored(root, path):
if (
path.is_symlink()
or not path.is_file()
or _has_ignored_dir(root, path)
or _is_ignored(root, path, patterns)
or _is_vendored(root, path)
):
continue
if detect_language(path) != "unknown":
paths.append(path)
@@ -92,3 +92,18 @@ calls = ["path.normalize", "path.resolve"]
# CWE-79. The standard encoders: the DOM/global URI encoders and the common escaping helpers.
id = "html.escape"
calls = ["encodeURIComponent", "encodeURI", "escapeHtml", "escape", "sanitizeHtml", "DOMPurify.sanitize"]
[[layout]]
# First-party test conventions are retained as graph context, ordered as unshipped.
# These declarations cover JS module/JSX and TS/TSX filenames; they do not assert
# TypeScript property support or a browser/server runtime classification.
id = "javascript_first_party_tests"
vendored = ["node_modules/"]
tests = [
"tests/", "test/", "spec/", "__tests__/", "__mocks__/", "e2e/",
"*.test.js", "*.test.mjs", "*.test.cjs", "*.test.jsx", "*.test.ts", "*.test.mts", "*.test.cts", "*.test.tsx",
"*.spec.js", "*.spec.mjs", "*.spec.cjs", "*.spec.jsx", "*.spec.ts", "*.spec.mts", "*.spec.cts", "*.spec.tsx",
"*.mock.js", "*.mock.mjs", "*.mock.cjs", "*.mock.jsx", "*.mock.ts", "*.mock.mts", "*.mock.cts", "*.mock.tsx",
"*.e2e.js", "*.e2e.mjs", "*.e2e.cjs", "*.e2e.jsx", "*.e2e.ts", "*.e2e.mts", "*.e2e.cts", "*.e2e.tsx",
]