mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
feat(pre-push-safety-net): exclude vendor inputs and partition frontend coverage
This commit is contained in:
@@ -4,7 +4,7 @@ import argparse
|
||||
import json
|
||||
import tempfile
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -903,6 +903,7 @@ def _model_payload(
|
||||
"scope": scope.to_payload() if scope is not None else None,
|
||||
"question_outcomes": [q.to_payload() for q in outcomes],
|
||||
"family_coverage": [c.to_payload() for c in getattr(result, "family_coverage", ())],
|
||||
"partitions": [asdict(p) for p in getattr(result, "partitions", ())],
|
||||
"by_rung": dict(getattr(result, "by_rung", {}) or {}),
|
||||
"regions_scanned": getattr(result, "regions_scanned", 0),
|
||||
"regions_unjudged": getattr(result, "regions_unjudged", 0),
|
||||
|
||||
@@ -697,9 +697,12 @@ class JoernBackend:
|
||||
methods = 0
|
||||
files = 0
|
||||
answered = 0
|
||||
file_names: set[str] = set()
|
||||
named_census = True
|
||||
for shard in shards:
|
||||
one = self.query_batch(shard, query, requests)
|
||||
if one is None:
|
||||
named_census = False
|
||||
if self.execution_budget is not None:
|
||||
self._note("shard_query_incomplete")
|
||||
return None
|
||||
@@ -709,11 +712,21 @@ class JoernBackend:
|
||||
if census and isinstance(census[0], Mapping):
|
||||
methods += int(str(census[0].get("methods", "0")) or 0)
|
||||
files += int(str(census[0].get("files", "0")) or 0)
|
||||
names = census[0].get("file_names")
|
||||
if isinstance(names, list) and all(isinstance(name, str) for name in names):
|
||||
file_names.update(names)
|
||||
else:
|
||||
named_census = False
|
||||
else:
|
||||
named_census = False
|
||||
for rid, rows in one.items():
|
||||
merged.setdefault(rid, []).extend(rows)
|
||||
if not answered:
|
||||
return None
|
||||
merged["__census__"] = [{"methods": str(methods), "files": str(files), "shards": str(len(shards))}]
|
||||
total: dict[str, object] = {"methods": str(methods), "files": str(files), "shards": str(len(shards))}
|
||||
if named_census:
|
||||
total["file_names"] = sorted(file_names)
|
||||
merged["__census__"] = [total]
|
||||
return merged
|
||||
|
||||
def _apply_overlays(self, cpg_path: Path, scratch: Path) -> bool:
|
||||
|
||||
@@ -78,7 +78,7 @@
|
||||
// per-file warnings -- so "no rows" and "no graph" are indistinguishable to the driver without this.
|
||||
// It rides the batch rather than costing its own invocation, because JVM startup is what this whole
|
||||
// batching design exists to avoid.
|
||||
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString))
|
||||
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString, "file_names" -> ujson.Arr.from(cpg.file.name.l)))
|
||||
println(ujson.write(ujson.Obj.from(answers.toSeq :+ ("__census__" -> census))))
|
||||
} else {
|
||||
println(ujson.write(ujson.Arr(rowsFor(settings, function, file): _*)))
|
||||
|
||||
@@ -190,7 +190,7 @@
|
||||
// per-file warnings -- so "no rows" and "no graph" are indistinguishable to the driver without this.
|
||||
// It rides the batch rather than costing its own invocation, because JVM startup is what this whole
|
||||
// batching design exists to avoid.
|
||||
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString))
|
||||
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString, "file_names" -> ujson.Arr.from(cpg.file.name.l)))
|
||||
println(ujson.write(ujson.Obj.from(answers.toSeq :+ ("__census__" -> census))))
|
||||
} else {
|
||||
println(ujson.write(ujson.Arr(rowsFor(operations, dischargers, function, file, operationRequires, dischargersByKind): _*)))
|
||||
|
||||
@@ -852,7 +852,7 @@
|
||||
// per-file warnings -- so "no rows" and "no graph" are indistinguishable to the driver without this.
|
||||
// It rides the batch rather than costing its own invocation, because JVM startup is what this whole
|
||||
// batching design exists to avoid.
|
||||
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString))
|
||||
val census = ujson.Arr(ujson.Obj("methods" -> cpg.method.size.toString, "files" -> cpg.file.size.toString, "file_names" -> ujson.Arr.from(cpg.file.name.l)))
|
||||
println(ujson.write(ujson.Obj.from(answers.toSeq :+ ("__census__" -> census))))
|
||||
} else {
|
||||
println(
|
||||
|
||||
@@ -26,7 +26,12 @@ def _matches(relative: str, patterns: Iterable[str]) -> bool:
|
||||
parts = relative.split("/")
|
||||
for pattern in patterns:
|
||||
if pattern.endswith("/"):
|
||||
if pattern.rstrip("/") in parts[:-1] or pattern.rstrip("/") in parts:
|
||||
directory = pattern.rstrip("/")
|
||||
if any(
|
||||
fnmatch.fnmatch("/".join(parts[start:end]), directory)
|
||||
for start in range(len(parts))
|
||||
for end in range(start + 1, len(parts) + 1)
|
||||
):
|
||||
return True
|
||||
elif fnmatch.fnmatch(relative, pattern) or fnmatch.fnmatch(parts[-1], pattern):
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Physical first-party frontend inputs and routing; selection remains in the scan ranker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..cpg.backend import _DeadlineExpired, _remove_owned_tree
|
||||
from ..preprocess import LANGUAGE_BY_EXTENSION
|
||||
from .contracts import ExecutionBudget
|
||||
from .layout import is_test_path, is_vendored, layout_facts
|
||||
|
||||
FRONTENDS = {
|
||||
"php": "php",
|
||||
"javascript": "javascript",
|
||||
"typescript": "typescript",
|
||||
"python": "python",
|
||||
"java": "java",
|
||||
"c": "c",
|
||||
"cpp": "c_cpp",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PartitionCoverage:
|
||||
language: str
|
||||
frontend: str | None
|
||||
paths: tuple[str, ...]
|
||||
source_bytes: int
|
||||
status: str
|
||||
boundaries: tuple[str, ...] = ()
|
||||
runtime: str = "unspecified"
|
||||
excluded_paths: tuple[str, ...] = ()
|
||||
unshipped_paths: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class PartitionGraph:
|
||||
"""Routes existing requests without reordering or allocating another scan budget."""
|
||||
|
||||
def __init__(self, budget: ExecutionBudget | None = None) -> None:
|
||||
self.budget = budget
|
||||
self.diagnostics: list[str] = []
|
||||
self.graphs: dict[str, Any] = {}
|
||||
self.roots: dict[str, Path] = {}
|
||||
self.path_languages: dict[str, str] = {}
|
||||
self.partitions: tuple[PartitionCoverage, ...] = ()
|
||||
self.boundaries: tuple[str, ...] = ()
|
||||
self.unparsed: tuple[str, ...] = ()
|
||||
self.cpg_path = Path("partitioned-cpg")
|
||||
|
||||
def cleanup(self) -> None:
|
||||
try:
|
||||
for graph in self.graphs.values():
|
||||
cleanup = getattr(graph, "cleanup", None)
|
||||
if callable(cleanup):
|
||||
cleanup()
|
||||
finally:
|
||||
for root in self.roots.values():
|
||||
deadline = self.budget.deadline_monotonic + self.budget.cancellation_allowance_seconds if self.budget else float("inf")
|
||||
try:
|
||||
_remove_owned_tree(root, deadline)
|
||||
except (OSError, _DeadlineExpired):
|
||||
self.diagnostics.append("scratch_cleanup_incomplete")
|
||||
|
||||
def execution_diagnostics(self) -> tuple[str, ...]:
|
||||
reasons = list(self.diagnostics)
|
||||
for graph in self.graphs.values():
|
||||
read = getattr(graph, "execution_diagnostics", None)
|
||||
if callable(read):
|
||||
reasons.extend(read())
|
||||
return tuple(reasons)
|
||||
|
||||
def _normalize(self, value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
for root in self.roots.values():
|
||||
value = value.replace(str(root) + "/", "")
|
||||
return value
|
||||
if isinstance(value, list):
|
||||
return [self._normalize(v) for v in value]
|
||||
if isinstance(value, dict):
|
||||
return {k: self._normalize(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
def run(self, kind: str, params: Mapping[str, object]) -> list[object] | None:
|
||||
return self.run_batch(kind, {"one": params}).get("one")
|
||||
|
||||
def run_batch(self, kind: str, requests: Mapping[str, Mapping[str, object]]) -> dict[str, list[object]]:
|
||||
answers: dict[str, list[object]] = {}
|
||||
languages = dict.fromkeys(self.path_languages.get(str(params.get("file", "")), "") for params in requests.values())
|
||||
for language in languages:
|
||||
graph = self.graphs.get(language)
|
||||
if graph is None:
|
||||
continue
|
||||
if self.budget is not None and time.monotonic() >= self.budget.deadline_monotonic:
|
||||
break
|
||||
selected = {rid: params for rid, params in requests.items() if self.path_languages.get(str(params.get("file", ""))) == language}
|
||||
if not selected:
|
||||
continue
|
||||
batch = getattr(graph, "run_batch", None)
|
||||
raw = batch(kind, selected) if callable(batch) else {rid: graph.run(kind, params) for rid, params in selected.items()}
|
||||
if not isinstance(raw, Mapping):
|
||||
continue
|
||||
for rid, rows in raw.items():
|
||||
if rid in selected and isinstance(rows, list):
|
||||
answers[rid] = self._normalize(rows)
|
||||
elif rid == "__census__" and isinstance(rows, list):
|
||||
expected = set(next(p.paths for p in self.partitions if p.language == language))
|
||||
normalized = self._normalize(rows)
|
||||
names = [r.get("file_names") for r in normalized if isinstance(r, Mapping)]
|
||||
named = bool(names) and all(isinstance(n, list) and all(isinstance(p, str) for p in n) for n in names)
|
||||
observed = {p.removeprefix("./") for n in names if isinstance(n, list) for p in n if isinstance(p, str)}
|
||||
if not named or not expected <= observed:
|
||||
self.diagnostics.append("partition_file_census_incomplete" if named else "partition_file_census_unavailable")
|
||||
rows = [
|
||||
{
|
||||
"methods": 0,
|
||||
"files": 0,
|
||||
"shards": max((int(str(r.get("shards", 1))) for r in rows if isinstance(r, Mapping)), default=1),
|
||||
}
|
||||
]
|
||||
# Preserve any individual empty or sharded graph's degradation.
|
||||
previous = answers.get(rid, [])
|
||||
if not previous or any(
|
||||
isinstance(r, Mapping) and (int(str(r.get("methods", 0))) == 0 or int(str(r.get("shards", 1))) > 1) for r in rows
|
||||
):
|
||||
answers[rid] = rows
|
||||
return answers
|
||||
|
||||
|
||||
def build_partitions(
|
||||
root: Path, build: Callable[[Path, str], Any], budget: ExecutionBudget | None, declared: Mapping[str, tuple[str, ...]] | None = None
|
||||
) -> PartitionGraph:
|
||||
graph = PartitionGraph(budget)
|
||||
paths: dict[str, list[Path]] = {}
|
||||
boundaries: set[str] = set()
|
||||
excluded: set[str] = set()
|
||||
facts = layout_facts()
|
||||
root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try:
|
||||
for directory, dirs, files, directory_fd in os.fwalk(
|
||||
".", follow_symlinks=False, dir_fd=root_fd, onerror=lambda error: boundaries.add("source_unreadable")
|
||||
):
|
||||
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
|
||||
boundaries.add("deadline_exhausted")
|
||||
break
|
||||
base = root / directory
|
||||
for name in list(dirs):
|
||||
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
|
||||
boundaries.add("deadline_exhausted")
|
||||
dirs.clear()
|
||||
break
|
||||
path = base / name
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if name in {".git", ".hg", ".svn", ".openultrasast", ".venv", "venv", "__pycache__"}:
|
||||
excluded.add(relative + "/")
|
||||
dirs.remove(name)
|
||||
continue
|
||||
if path.is_symlink() or is_vendored(relative + "/", facts):
|
||||
excluded.add(relative + "/")
|
||||
boundaries.add("symlink_context_unresolved" if path.is_symlink() else "vendor_semantics_unresolved")
|
||||
dirs.remove(name)
|
||||
for name in files:
|
||||
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
|
||||
boundaries.add("deadline_exhausted")
|
||||
break
|
||||
path = base / name
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if is_vendored(relative, facts):
|
||||
excluded.add(relative)
|
||||
boundaries.add("vendor_semantics_unresolved")
|
||||
continue
|
||||
if path.is_symlink() or not path.is_file():
|
||||
excluded.add(relative)
|
||||
boundaries.add("symlink_context_unresolved")
|
||||
continue
|
||||
language = LANGUAGE_BY_EXTENSION.get(path.suffix.lower(), "unknown")
|
||||
if language == "unknown":
|
||||
try:
|
||||
fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory_fd)
|
||||
with os.fdopen(fd, "rb") as handle:
|
||||
first = handle.readline(4096).lower()
|
||||
except OSError:
|
||||
boundaries.add("source_unreadable")
|
||||
continue
|
||||
if first.startswith(b"#!"):
|
||||
language = "python" if b"python" in first else "javascript" if b"node" in first else "unknown"
|
||||
languages = (language,) if language != "unknown" else (declared or {}).get(relative, ())
|
||||
for item in languages:
|
||||
paths.setdefault(item, []).append(path)
|
||||
if len(languages) == 1:
|
||||
graph.path_languages[relative] = languages[0]
|
||||
elif languages:
|
||||
boundaries.add("ambiguous_frontend_path")
|
||||
if len(paths) > 1:
|
||||
boundaries.add("cross_partition_semantics_unresolved")
|
||||
records: list[PartitionCoverage] = []
|
||||
for language, sources in sorted(paths.items()):
|
||||
relative_paths = tuple(sorted(p.relative_to(root).as_posix() for p in sources))
|
||||
frontend = FRONTENDS.get(language)
|
||||
if frontend is None:
|
||||
records.append(PartitionCoverage(language, None, relative_paths, 0, "unsupported", ("frontend_unsupported",)))
|
||||
boundaries.add("frontend_unsupported")
|
||||
continue
|
||||
scratch = Path(tempfile.mkdtemp(prefix="ousast-input-"))
|
||||
graph.roots[language] = scratch
|
||||
size = 0
|
||||
complete = True
|
||||
for source in sources:
|
||||
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
|
||||
complete = False
|
||||
break
|
||||
target = scratch / source.relative_to(root)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
# A no-follow open prevents file symlink substitution between census and read.
|
||||
try:
|
||||
fd = _open_source(root_fd, source.relative_to(root).parts)
|
||||
except OSError:
|
||||
boundaries.add("source_unreadable")
|
||||
complete = False
|
||||
break
|
||||
with os.fdopen(fd, "rb") as handle, target.open("wb") as output:
|
||||
while chunk := handle.read(1024 * 1024):
|
||||
if budget is not None and time.monotonic() >= budget.deadline_monotonic:
|
||||
complete = False
|
||||
break
|
||||
output.write(chunk)
|
||||
size += len(chunk)
|
||||
if not complete:
|
||||
break
|
||||
built = build(scratch, frontend) if complete else None
|
||||
if built is not None:
|
||||
graph.graphs[language] = built
|
||||
graph.unparsed += tuple(graph._normalize(list(getattr(built, "unparsed", ()))))
|
||||
record_limits: tuple[str, ...] = ("runtime_unspecified",) if language in {"javascript", "typescript"} else ()
|
||||
if language == "typescript":
|
||||
record_limits += ("typescript_property_support_unvalidated",)
|
||||
boundaries.add("typescript_property_support_unvalidated")
|
||||
if language in {"c", "cpp"}:
|
||||
record_limits += ("c_bounds_arithmetic_unmodeled",)
|
||||
records.append(
|
||||
PartitionCoverage(language, frontend, relative_paths, size, "built" if built is not None else "build_failed", record_limits)
|
||||
)
|
||||
from dataclasses import replace
|
||||
|
||||
graph.partitions = tuple(
|
||||
replace(
|
||||
record, excluded_paths=tuple(sorted(excluded)), unshipped_paths=tuple(p for p in record.paths if is_test_path(p, facts))
|
||||
)
|
||||
for record in records
|
||||
)
|
||||
graph.boundaries = tuple(sorted(boundaries))
|
||||
return graph
|
||||
except BaseException:
|
||||
graph.cleanup()
|
||||
raise
|
||||
finally:
|
||||
os.close(root_fd)
|
||||
|
||||
|
||||
def _open_source(root_fd: int, parts: tuple[str, ...]) -> int:
|
||||
descriptor = os.dup(root_fd)
|
||||
try:
|
||||
for component in parts[:-1]:
|
||||
child = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor)
|
||||
os.close(descriptor)
|
||||
descriptor = child
|
||||
result = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=descriptor)
|
||||
if not stat.S_ISREG(os.fstat(result).st_mode):
|
||||
os.close(result)
|
||||
raise OSError("source is not a regular file")
|
||||
return result
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
@@ -43,7 +43,8 @@ from .contracts import (
|
||||
from .dominance import request_params as dominance_params
|
||||
from .evidence import TIER_EXCLUDE, Evidence, evidence_from_rows
|
||||
from .ladder import Rung
|
||||
from .layout import with_layout
|
||||
from .layout import is_vendored, layout_facts, with_layout
|
||||
from .partitions import PartitionCoverage, build_partitions
|
||||
from .pipeline import ModelFinding, scan_region
|
||||
from .regions import MODULE_SCOPE, ScanRegion, affected_context
|
||||
from .specs import ConfigSpec, DominanceSpec, TaintSpec, config_specs, dominance_specs, taint_specs
|
||||
@@ -126,6 +127,7 @@ class ModelScanResult:
|
||||
# could have been found was skipped. Reported so the request count is legible.
|
||||
requests_pruned: int = 0
|
||||
|
||||
partitions: tuple[PartitionCoverage, ...] = ()
|
||||
change_context: ChangeContext | None = None
|
||||
scope: ScopeDecision | None = None
|
||||
question_outcomes: tuple[QuestionOutcome, ...] = ()
|
||||
@@ -158,6 +160,8 @@ def scan_repository(
|
||||
if ranking_mode is not None:
|
||||
limits = replace(limits, order_by_evidence=ranking_mode == "evidence")
|
||||
mode = "evidence" if limits.order_by_evidence else "static"
|
||||
layout = layout_facts()
|
||||
regions = tuple(r for r in regions if not is_vendored(r.path, layout))
|
||||
# Reject duplicate canonical questions before any engine work or resource ownership.
|
||||
identities = [_identity(unit, r, f) for r in regions for f in r.families]
|
||||
if len(set(identities)) != len(identities):
|
||||
@@ -223,13 +227,29 @@ def _scan_repository_impl(
|
||||
for region in regions:
|
||||
counts[region.language] = counts.get(region.language, 0) + 1
|
||||
dominant = max(counts, key=lambda name: (counts[name], name)) if counts else ""
|
||||
cpg = _build(backend, root, dominant, execution_budget=execution_budget)
|
||||
declarations: dict[str, set[str]] = {}
|
||||
for region in regions:
|
||||
declarations.setdefault(region.path, set()).add(region.language)
|
||||
if root.is_dir():
|
||||
cpg = build_partitions(
|
||||
root,
|
||||
lambda source, language: _build(backend, source, language, execution_budget=execution_budget),
|
||||
execution_budget,
|
||||
{path: tuple(sorted(languages)) for path, languages in declarations.items()},
|
||||
)
|
||||
degradations.extend({"stage": "model", "reason": reason} for reason in cpg.boundaries)
|
||||
else:
|
||||
cpg = _build(backend, root, dominant, execution_budget=execution_budget)
|
||||
build_seconds = round(time.monotonic() - build_started, 2)
|
||||
if cpg is None:
|
||||
if cpg is None or (hasattr(cpg, "graphs") and not cpg.graphs):
|
||||
if cpg is not None:
|
||||
owned.append(cpg)
|
||||
scope, _ = _scope_work(
|
||||
regions, {}, limits, unit, ranking_mode, population_complete, failure="cpg_build_failed", change_context=change_context
|
||||
)
|
||||
scope = replace(scope, unresolved_boundaries=tuple(dict.fromkeys((*scope.unresolved_boundaries, *getattr(cpg, "boundaries", ())))))
|
||||
return ModelScanResult(
|
||||
partitions=getattr(cpg, "partitions", ()),
|
||||
change_context=change_context,
|
||||
scope=scope,
|
||||
family_coverage=_family_coverage(scope, ()),
|
||||
@@ -322,7 +342,22 @@ def _scan_repository_impl(
|
||||
# Stable question IDs are the IDs sent to every execution request below.
|
||||
scope_limits = (
|
||||
replace(limits, prune_tier0=False)
|
||||
if any(d.get("reason") in {"files_unparsed", "cpg_empty", "cpg_sharded"} for d in degradations)
|
||||
if any(
|
||||
d.get("reason")
|
||||
in {
|
||||
"files_unparsed",
|
||||
"cpg_empty",
|
||||
"cpg_sharded",
|
||||
"cross_partition_semantics_unresolved",
|
||||
"vendor_semantics_unresolved",
|
||||
"symlink_context_unresolved",
|
||||
"frontend_unsupported",
|
||||
"source_unreadable",
|
||||
"ambiguous_frontend_path",
|
||||
"typescript_property_support_unvalidated",
|
||||
}
|
||||
for d in degradations
|
||||
)
|
||||
else limits
|
||||
)
|
||||
scope, work = _scope_work(
|
||||
@@ -457,16 +492,32 @@ def _scan_repository_impl(
|
||||
if unasked:
|
||||
degradations.append({"stage": "model", "reason": "budget_exhausted", "regions_unasked": len(unasked)})
|
||||
|
||||
graph_gaps = tuple(str(d["reason"]) for d in degradations if d.get("reason") in {"files_unparsed", "cpg_empty", "cpg_sharded"})
|
||||
graph_gaps = tuple(
|
||||
str(d["reason"])
|
||||
for d in degradations
|
||||
if d.get("reason")
|
||||
in {
|
||||
"files_unparsed",
|
||||
"cpg_empty",
|
||||
"cpg_sharded",
|
||||
"cross_partition_semantics_unresolved",
|
||||
"vendor_semantics_unresolved",
|
||||
"symlink_context_unresolved",
|
||||
"frontend_unsupported",
|
||||
"source_unreadable",
|
||||
"ambiguous_frontend_path",
|
||||
"typescript_property_support_unvalidated",
|
||||
}
|
||||
)
|
||||
for question in scope.selected:
|
||||
rid = question.identity.question_id
|
||||
if rid not in outcomes:
|
||||
answered = rid in rows_by_id
|
||||
has_answer = rid in rows_by_id
|
||||
outcomes[rid] = QuestionOutcome(
|
||||
question.identity,
|
||||
"not_arbitrated" if answered else "unanswered",
|
||||
"not_arbitrated" if has_answer else "unanswered",
|
||||
"deadline_exhausted" if not _within_deadline(execution_budget) else "query_unanswered",
|
||||
_rows_json(rows_by_id[rid]) if answered else None,
|
||||
_rows_json(rows_by_id[rid]) if has_answer else None,
|
||||
)
|
||||
if question.identity in context_gaps and outcomes[rid].status == "completed":
|
||||
outcomes[rid] = replace(outcomes[rid], status="unresolved", reason="change_context_incomplete")
|
||||
@@ -480,6 +531,7 @@ def _scan_repository_impl(
|
||||
# The graph has answered everything it is going to. Its scratch tree holds the CPG, the request files and
|
||||
# joern's own working copy -- tens of megabytes per scan -- and nothing else reclaims it.
|
||||
return ModelScanResult(
|
||||
partitions=getattr(cpg, "partitions", ()),
|
||||
findings=findings,
|
||||
by_rung=_tally(findings),
|
||||
regions_scanned=scanned,
|
||||
|
||||
@@ -97,7 +97,13 @@ def enumerate_source_files(root: Path) -> list[Path]:
|
||||
patterns = _load_ignore_patterns(root)
|
||||
paths: list[Path] = []
|
||||
for path in root.rglob("*"):
|
||||
if not path.is_file() or _has_ignored_dir(root, path) or _is_ignored(root, path, patterns) or _is_vendored(root, path):
|
||||
if (
|
||||
path.is_symlink()
|
||||
or not path.is_file()
|
||||
or _has_ignored_dir(root, path)
|
||||
or _is_ignored(root, path, patterns)
|
||||
or _is_vendored(root, path)
|
||||
):
|
||||
continue
|
||||
if detect_language(path) != "unknown":
|
||||
paths.append(path)
|
||||
|
||||
@@ -92,3 +92,18 @@ calls = ["path.normalize", "path.resolve"]
|
||||
# CWE-79. The standard encoders: the DOM/global URI encoders and the common escaping helpers.
|
||||
id = "html.escape"
|
||||
calls = ["encodeURIComponent", "encodeURI", "escapeHtml", "escape", "sanitizeHtml", "DOMPurify.sanitize"]
|
||||
|
||||
|
||||
[[layout]]
|
||||
# First-party test conventions are retained as graph context, ordered as unshipped.
|
||||
# These declarations cover JS module/JSX and TS/TSX filenames; they do not assert
|
||||
# TypeScript property support or a browser/server runtime classification.
|
||||
id = "javascript_first_party_tests"
|
||||
vendored = ["node_modules/"]
|
||||
tests = [
|
||||
"tests/", "test/", "spec/", "__tests__/", "__mocks__/", "e2e/",
|
||||
"*.test.js", "*.test.mjs", "*.test.cjs", "*.test.jsx", "*.test.ts", "*.test.mts", "*.test.cts", "*.test.tsx",
|
||||
"*.spec.js", "*.spec.mjs", "*.spec.cjs", "*.spec.jsx", "*.spec.ts", "*.spec.mts", "*.spec.cts", "*.spec.tsx",
|
||||
"*.mock.js", "*.mock.mjs", "*.mock.cjs", "*.mock.jsx", "*.mock.ts", "*.mock.mts", "*.mock.cts", "*.mock.tsx",
|
||||
"*.e2e.js", "*.e2e.mjs", "*.e2e.cjs", "*.e2e.jsx", "*.e2e.ts", "*.e2e.mts", "*.e2e.cts", "*.e2e.tsx",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user