feat(pre-push-safety-net): exclude vendor inputs and partition frontend coverage

This commit is contained in:
Marius Ciepluch
2026-09-13 11:53:29 +02:00
parent 51fabef9ea
commit 6ba2b06353
29 changed files with 1996 additions and 72 deletions
+30
View File
@@ -956,6 +956,36 @@ def test_a_source_file_the_size_of_a_data_table_is_excluded_and_named(tmp_path:
assert [Path(name).name for name in result.unparsed] == ["profiles.php"], "and it is reported, not silently dropped"
@pytest.mark.parametrize("shard_names", [("app.php",), ("app.php", "helper.php")])
def test_shard_aggregation_preserves_complete_named_census(monkeypatch, shard_names):
from openultrasast.cpg.backend import JoernBackend
def batch(self, path, query, requests):
return {"__census__": [{"methods": "2", "files": "2", "file_names": ["<unknown>", path.name]}]}
monkeypatch.setattr(JoernBackend, "query_batch", batch)
result = JoernBackend().query_batch_across([Path(name) for name in shard_names], "taint", {})
assert result is not None
assert set(result["__census__"][0]["file_names"]) == {"<unknown>", *shard_names}
assert result["__census__"][0]["shards"] == str(len(shard_names))
@pytest.mark.parametrize("missing", ["names", "census", "answer"])
def test_shard_aggregation_does_not_invent_complete_named_census(monkeypatch, missing):
from openultrasast.cpg.backend import JoernBackend
def batch(self, path, query, requests):
if path.name == "missing":
return None if missing == "answer" else {} if missing == "census" else {"__census__": [{"methods": "2", "files": "2"}]}
return {"__census__": [{"methods": "2", "files": "2", "file_names": ["app.php", "<unknown>"]}]}
monkeypatch.setattr(JoernBackend, "query_batch", batch)
result = JoernBackend().query_batch_across([Path("app.php"), Path("missing")], "taint", {})
assert result is not None
assert "file_names" not in result["__census__"][0]
assert result["__census__"][0]["shards"] == "2"
def test_a_clean_build_whose_census_cannot_be_taken_is_a_failed_build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""`cpg query census failed: timeout` was logged as a warning during a build whose every later query
then timed out too. A graph that cannot answer the census inside the query timeout cannot answer a taint
+3
View File
@@ -261,6 +261,7 @@ def test_cleanup_does_not_follow_substituted_directory_links(tmp_path: Path) ->
def test_driver_propagates_budget_and_rejects_legacy_without_launch(tmp_path: Path) -> None:
(tmp_path / "input.py").write_bytes(b"print(1)\n")
from openultrasast.cpg.backend import CpgResult
from openultrasast.model.scan import scan_repository
@@ -296,6 +297,7 @@ def test_driver_propagates_budget_and_rejects_legacy_without_launch(tmp_path: Pa
def test_budgeted_driver_does_not_retry_internal_typeerror(tmp_path: Path) -> None:
(tmp_path / "input.py").write_bytes(b"print(1)\n")
from openultrasast.model.scan import scan_repository
class Backend:
@@ -337,6 +339,7 @@ def test_cancelled_graph_session_does_not_poison_next_build(tmp_path: Path, monk
def test_driver_cleans_graph_on_unexpected_exception(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
(tmp_path / "input.py").write_bytes(b"print(1)\n")
from openultrasast.cpg.backend import CpgResult
from openultrasast.model import scan
+6 -2
View File
@@ -65,11 +65,13 @@ def test_test_regions_are_not_product_and_order_last() -> None:
)
def test_the_scan_hands_the_vendored_trees_to_the_build(tmp_path: Path) -> None:
def test_the_scan_excludes_vendored_trees_before_the_build(tmp_path: Path) -> None:
from openultrasast.model.regions import ScanRegion
from openultrasast.model.scan import ScanBudget, scan_repository
(tmp_path / "vendor").mkdir()
(tmp_path / "a.php").write_bytes(b"<?php echo 1;")
(tmp_path / "vendor/b.php").write_bytes(b"<?php vendor_secret();")
seen: dict[str, object] = {}
class _Backend:
@@ -80,8 +82,10 @@ def test_the_scan_hands_the_vendored_trees_to_the_build(tmp_path: Path) -> None:
from openultrasast.cpg.backend import CpgResult
seen["exclude"] = tuple(exclude)
seen["files"] = tuple(p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file())
return CpgResult(cpg_path=Path("cpg.bin"), run=lambda q, p: [], run_batch=lambda q, r: {rid: [] for rid in r})
region = ScanRegion(path="a.php", function="f", language="php", families=("injection",), rank=0.5, source="x")
scan_repository(tmp_path, [region], backend=_Backend(), budget=ScanBudget(max_model_calls=0))
assert seen["exclude"] == ("vendor",)
assert seen["exclude"] == ()
assert seen["files"] == ("a.php",)
+337
View File
@@ -0,0 +1,337 @@
from pathlib import Path
from openultrasast.cpg.backend import CpgResult
from openultrasast.model.regions import ScanRegion
from openultrasast.model.scan import ScanBudget, scan_repository
def test_polyglot_graph_inputs_and_single_scope(tmp_path: Path) -> None:
sources = {
"api.php": "<?php function f($x) { echo $x; }",
"api.js": "function f(x) { return x; }",
"vendor/bad.php": "<?php echo 1;",
"node_modules/bad.js": "bad()",
"tests/test.js": "test()",
"other.rs": "fn main() {}",
}
for name, content in sources.items():
path = tmp_path / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
seen = []
class Backend:
def build(self, root, *, language="", **kwargs):
files = tuple(sorted(p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file()))
assert all(p.read_bytes() for p in root.rglob("*") if p.is_file())
seen.append((language, files))
return CpgResult(Path("cpg.bin"), lambda q, p: [], run_batch=lambda q, requests: {rid: [] for rid in requests})
regions = [
ScanRegion(path=p, function="f", language=language, families=("injection",), rank=r, source="entry_point")
for p, language, r in [
("api.php", "php", 1.0),
("api.js", "javascript", 0.9),
("vendor/bad.php", "php", 2.0),
("tests/test.js", "javascript", 3.0),
]
]
result = scan_repository(
tmp_path, regions, backend=Backend(), budget=ScanBudget(max_regions=1, tiering=False), population_complete=True
)
assert seen == [("javascript", ("api.js", "tests/test.js")), ("php", ("api.php",))]
assert result.scope.selected[0].identity.path == "api.php"
assert all(q.identity.path != "vendor/bad.php" for q in (*result.scope.selected, *result.scope.deferred))
assert any(p.language == "rust" and p.status == "unsupported" for p in result.partitions)
assert "cross_partition_semantics_unresolved" in result.scope.unresolved_boundaries
def test_file_patterns_symlinks_and_witness_paths(tmp_path, monkeypatch):
import dataclasses
from openultrasast.model import layout, partitions
from openultrasast.preprocess import preprocess_repository
from openultrasast.semantic.facts import LayoutFact
facts = (LayoutFact(id="declared", language="javascript", vendored=("*.bundle.js", "bundled/lib/"), tests=("tests/",)),)
monkeypatch.setattr(layout, "layout_facts", lambda *args: facts)
monkeypatch.setattr(partitions, "layout_facts", lambda *args: facts)
outside = tmp_path.parent / "external.js"
outside.write_text("external_secret()")
(tmp_path / "escape.js").symlink_to(outside)
(tmp_path / "bad.bundle.js").write_text("vendor_secret()")
(tmp_path / "app.js").write_text("function f(x) { return x; }")
(tmp_path / "bundled/lib").mkdir(parents=True)
(tmp_path / "bundled/lib/hidden.js").write_text("vendor_secret()")
_, targets = preprocess_repository(tmp_path)
assert [t.path for t in targets] == ["app.js"]
roots = []
def build(root, language):
roots.append(root)
assert [p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file()] == ["app.js"]
return CpgResult(
Path("graph"),
lambda q, p: [],
run_batch=lambda q, reqs: {rid: [{"file": str(root / "app.js"), "site": str(root / "app.js") + ":1"}] for rid in reqs},
)
graph = partitions.build_partitions(tmp_path, build, None)
try:
assert graph.partitions[0].source_bytes == len((tmp_path / "app.js").read_bytes())
assert graph.run("taint", {"file": "app.js"}) == [{"file": "app.js", "site": "app.js:1"}]
assert {"vendor_semantics_unresolved", "symlink_context_unresolved"} <= set(graph.boundaries)
assert dataclasses.asdict(graph.partitions[0])["runtime"] == "unspecified"
finally:
graph.cleanup()
assert all(not root.exists() for root in roots)
def test_shared_deadline_stops_later_frontend_and_no_false_clean(tmp_path):
import time
from openultrasast.model.contracts import ExecutionBudget
from openultrasast.model.partitions import build_partitions
for name in ("a.js", "b.php"):
(tmp_path / name).write_text("readable")
seen = []
budget = ExecutionBudget(time.monotonic() + 0.1, 0.1)
def build(root, language):
seen.append(language)
time.sleep(0.12)
return CpgResult(Path("graph"), lambda q, p: [])
graph = build_partitions(tmp_path, build, budget)
try:
assert seen == ["javascript"]
assert [p.status for p in graph.partitions] == ["built", "build_failed"]
assert graph.run("taint", {"file": "b.php"}) is None
finally:
graph.cleanup()
def test_evidence_ranker_selects_once_across_languages(tmp_path):
from test_model_scope import vector
for name in ("a.php", "b.js"):
(tmp_path / name).write_text("readable source")
executed = []
class Backend:
def build(self, root, *, language="", **kwargs):
def batch(kind, requests):
if any(p.get("evidenceOnly") for p in requests.values()):
return {rid: vector(language == "javascript") for rid in requests}
executed.extend(p["file"] for p in requests.values())
return {rid: [] for rid in requests}
return CpgResult(Path("graph"), lambda q, p: [], run_batch=batch)
regions = [
ScanRegion(path=path, function="f", language=language, families=("injection",), rank=rank, source="entry_point")
for path, language, rank in [("a.php", "php", 1.0), ("b.js", "javascript", 0.5)]
]
result = scan_repository(tmp_path, regions, backend=Backend(), ranking_mode="evidence", budget=ScanBudget(max_regions=1))
assert len(result.scope.selected) == 1
assert result.scope.selected[0].identity.path == "b.js"
assert executed == ["b.js"]
assert result.question_outcomes[0].status == "unresolved"
def test_unsupported_only_population_is_named(tmp_path):
(tmp_path / "main.go").write_text("package main")
class Backend:
def build(self, *args, **kwargs):
raise AssertionError("unsupported frontend must not build")
result = scan_repository(tmp_path, [], backend=Backend(), population_complete=True)
assert result.partitions[0].language == "go"
assert result.partitions[0].status == "unsupported"
assert "frontend_unsupported" in result.scope.unresolved_boundaries
def test_partial_partition_census_never_completes_questions(tmp_path):
for name in ("a.py", "b.py"):
(tmp_path / name).write_text("print('readable')")
class Backend:
def build(self, root, **kwargs):
return CpgResult(
Path("graph"),
lambda q, p: [],
run_batch=lambda q, reqs: {**{rid: [] for rid in reqs}, "__census__": [{"methods": 2, "files": 1}]},
)
region = ScanRegion(path="a.py", function="f", language="python", families=("injection",), rank=1.0, source="entry_point")
result = scan_repository(tmp_path, [region], backend=Backend(), budget=ScanBudget(tiering=False))
assert result.question_outcomes[0].status == "unresolved"
assert "cpg_empty" in result.scope.unresolved_boundaries
def test_projection_refuses_substituted_ancestor(tmp_path):
import os
import pytest
from openultrasast.model.partitions import _open_source
source = tmp_path / "source"
source.mkdir()
outside = tmp_path / "outside"
outside.mkdir()
(outside / "secret.js").write_bytes(b"must not enter the graph")
(source / "nested").symlink_to(outside, target_is_directory=True)
descriptor = os.open(source, os.O_RDONLY | os.O_DIRECTORY)
try:
with pytest.raises(OSError):
_open_source(descriptor, ("nested", "secret.js"))
finally:
os.close(descriptor)
def test_partition_records_reach_cli(tmp_path):
from openultrasast.cli import _model_payload
(tmp_path / "main.go").write_bytes(b"package main")
result = scan_repository(tmp_path, [], backend=object())
payload = _model_payload(result)
assert payload["partitions"][0]["language"] == "go"
assert payload["partitions"][0]["status"] == "unsupported"
def test_stale_region_cannot_bypass_vendor_graph_exclusion(tmp_path):
(tmp_path / "api.php").write_bytes(b"<?php echo 1;")
(tmp_path / "vendor").mkdir()
(tmp_path / "vendor/bad.php").write_bytes(b"<?php vendor_secret();")
seen = []
class Backend:
def build(self, root, **kwargs):
seen.extend(p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file())
return CpgResult(Path("graph"), lambda q, p: [])
region = ScanRegion(path="missing.php", function="f", language="php", families=("injection",), rank=1.0, source="entry_point")
result = scan_repository(tmp_path, [region], backend=Backend())
assert seen == ["api.php"]
assert result.question_outcomes[0].status == "unanswered"
def test_unreadable_unknown_extension_reports_incomplete_without_crashing(tmp_path, monkeypatch):
from openultrasast.model import partitions
(tmp_path / "app.js").write_bytes(b"function handler(value) { return value; }\n")
(tmp_path / "README.md").write_bytes(b"unreadable declaration candidate\n")
original_open = partitions.os.open
def unreadable_open(path, flags, *args, **kwargs):
if str(path) == "README.md":
raise PermissionError("controlled unreadable discovery input")
return original_open(path, flags, *args, **kwargs)
monkeypatch.setattr(partitions.os, "open", unreadable_open)
class Backend:
def build(self, root, **kwargs):
assert (root / "app.js").read_bytes()
return CpgResult(Path("graph"), lambda query, params: [])
region = ScanRegion("app.js", "handler", "javascript", ("injection",), 1.0, "entry_point")
result = scan_repository(tmp_path, [region], backend=Backend(), budget=ScanBudget(tiering=False))
assert result.partitions[0].status == "built"
assert "source_unreadable" in result.scope.unresolved_boundaries
assert result.question_outcomes[0].status == "unresolved"
def test_declared_javascript_tests_remain_inputs_but_unshipped(tmp_path):
from openultrasast.model.layout import with_layout
from openultrasast.model.partitions import build_partitions
names = ("unit.test.js", "unit.spec.mjs", "unit.mock.cjs", "unit.e2e.jsx", "unit.test.ts", "unit.spec.tsx", "__tests__/unit.js")
for name in (*names, "app.js"):
source = tmp_path / name
source.parent.mkdir(parents=True, exist_ok=True)
source.write_text("function readable() {}")
graph = build_partitions(tmp_path, lambda root, language: CpgResult(Path("graph"), lambda q, p: []), None)
try:
assert set(names) <= {p for record in graph.partitions for p in record.paths}
assert set(names) == {p for record in graph.partitions for p in record.unshipped_paths}
regions = [
ScanRegion(path=p, function="readable", language="javascript", families=("injection",), rank=1.0, source="entry_point")
for p in (*names, "app.js")
]
assert {r.path for r in with_layout(regions) if not r.shipped} == set(names)
finally:
graph.cleanup()
def test_synthetic_census_node_cannot_mask_missing_source(tmp_path):
from openultrasast.model.partitions import build_partitions
for name in ("a.php", "b.php"):
(tmp_path / name).write_text("<?php echo 1;")
def build(root, language):
return CpgResult(
Path("graph"),
lambda q, p: [],
run_batch=lambda q, reqs: {
**{rid: [] for rid in reqs},
"__census__": [{"methods": 3, "files": 2, "file_names": ["<unknown>", str(root / "a.php")]}],
},
)
graph = build_partitions(tmp_path, build, None)
try:
result = graph.run_batch("taint", {"q": {"file": "a.php"}})
assert result["__census__"][0]["files"] == 0
assert "partition_file_census_incomplete" in graph.execution_diagnostics()
finally:
graph.cleanup()
def test_named_census_accepts_exact_paths_for_every_query_kind(tmp_path):
from openultrasast.model.partitions import build_partitions
(tmp_path / "app.php").write_text("<?php echo 1;")
def build(root, language):
return CpgResult(
Path("graph"),
lambda q, p: [],
run_batch=lambda q, reqs: {
**{rid: [] for rid in reqs},
"__census__": [{"methods": 2, "files": 2, "file_names": ["<unknown>", str(root / "app.php")]}],
},
)
graph = build_partitions(tmp_path, build, None)
try:
for kind in ("taint", "dominance", "config"):
assert graph.run_batch(kind, {"q": {"file": "app.php"}})["__census__"][0]["files"] == 2
assert not graph.execution_diagnostics()
finally:
graph.cleanup()
def test_unnamed_sharded_census_remains_explicitly_incomplete(tmp_path):
from openultrasast.model.partitions import build_partitions
(tmp_path / "app.php").write_text("<?php echo 1;")
graph = build_partitions(
tmp_path,
lambda root, language: CpgResult(
Path("graph"),
lambda q, p: [],
run_batch=lambda q, reqs: {**{rid: [] for rid in reqs}, "__census__": [{"methods": 20, "files": 20, "shards": 2}]},
),
None,
)
try:
result = graph.run_batch("taint", {"q": {"file": "app.php"}})
assert result["__census__"] == [{"methods": 0, "files": 0, "shards": 2}]
assert "partition_file_census_unavailable" in graph.execution_diagnostics()
finally:
graph.cleanup()
+12 -7
View File
@@ -394,7 +394,7 @@ def test_a_query_the_engine_could_not_answer_is_recorded_not_ignored() -> None:
assert result.findings == ()
def test_a_region_cap_that_hides_most_of_a_repository_is_reported() -> None:
def test_a_region_cap_that_hides_most_of_a_repository_is_reported(tmp_path: Path) -> None:
"""Paid Memberships Pro yields 4,463 regions from 637 files. A default budget of 500 examines 11% of it,
and a report that does not say so invites its silence to be read as a clean bill of health for the rest.
"""
@@ -414,14 +414,16 @@ def test_a_region_cap_that_hides_most_of_a_repository_is_reported() -> None:
ScanRegion(path=f"f{n}.py", function="handler", language="python", families=("injection",), rank=0.5, source="entry_point")
for n in range(30)
]
result = scan_repository(Path("."), regions, backend=_Backend(), budget=ScanBudget(max_model_calls=0, max_regions=10))
for region in regions:
(tmp_path / region.path).write_bytes(b"print(1)\n")
result = scan_repository(tmp_path, regions, backend=_Backend(), budget=ScanBudget(max_model_calls=0, max_regions=10))
truncated = [d for d in result.degradations if d.get("reason") == "regions_truncated"]
assert truncated, "the cap hid two thirds of the regions and said nothing"
assert truncated[0]["examined"] == 10 and truncated[0]["total"] == 30
def test_tier_zero_pairs_are_not_asked_and_nothing_else_is_skipped() -> None:
def test_tier_zero_pairs_are_not_asked_and_nothing_else_is_skipped(tmp_path: Path) -> None:
"""Phase 1 of flow-aware-ranking. A pair with no sink of its family in reach cannot yield a flow, so its
taint request is never issued -- and every other pair's request still is."""
from openultrasast.model.regions import ScanRegion
@@ -466,7 +468,8 @@ def test_tier_zero_pairs_are_not_asked_and_nothing_else_is_skipped() -> None:
region = ScanRegion(
path="a.py", function="run", language="python", families=("injection", "path", "deserialization"), rank=0.9, source="entry_point"
)
result = scan_repository(Path("."), [region], backend=_Backend(), budget=ScanBudget(max_model_calls=0))
(tmp_path / "a.py").write_bytes(b"print(1)\n")
result = scan_repository(tmp_path, [region], backend=_Backend(), budget=ScanBudget(max_model_calls=0))
taint = next(i for i in issued if i["kind"] == "taint")
assert taint["count"] == 1, "only the pair with a sink in reach is asked"
@@ -474,7 +477,7 @@ def test_tier_zero_pairs_are_not_asked_and_nothing_else_is_skipped() -> None:
assert not any(d.get("reason") == "query_failed" for d in result.degradations)
def test_the_budget_is_spent_by_evidence_when_asked() -> None:
def test_the_budget_is_spent_by_evidence_when_asked(tmp_path: Path) -> None:
"""Phase 2 of flow-aware-ranking. With `order_by_evidence`, the evidence pass runs over every region and
the region cap is cut from the (tier, score) order, so a low-ranked region with an open sink and a
local source is examined before a top-ranked region with no sink in reach. Off, the static order holds."""
@@ -523,15 +526,17 @@ def test_the_budget_is_spent_by_evidence_when_asked() -> None:
top = ScanRegion(path="top.py", function="handle", language="python", families=("injection",), rank=1.0, source="entry_point")
low = ScanRegion(path="low.py", function="run", language="python", families=("injection",), rank=0.3, source="entry_point")
(tmp_path / "top.py").write_bytes(b"print(1)\n")
(tmp_path / "low.py").write_bytes(b"print(2)\n")
static = _Backend()
scan_repository(Path("."), [top, low], backend=static, budget=ScanBudget(max_model_calls=0, max_regions=1))
scan_repository(tmp_path, [top, low], backend=static, budget=ScanBudget(max_model_calls=0, max_regions=1))
assert static.evidence_sizes == [1] and static.taint_files == [], (
"static order: the budget holds `top`, whose pair is tier 0 and pruned"
)
ordered = _Backend()
result = scan_repository(
Path("."), [top, low], backend=ordered, budget=ScanBudget(max_model_calls=0, max_regions=1, order_by_evidence=True)
tmp_path, [top, low], backend=ordered, budget=ScanBudget(max_model_calls=0, max_regions=1, order_by_evidence=True)
)
assert ordered.evidence_sizes == [2], "the evidence pass covers every region, not only the budget"
assert ordered.taint_files == [["low.py"]], "and the budget is spent on the region the evidence points at"
+3 -3
View File
@@ -23,7 +23,7 @@ class Backend:
def build(self, root, **kwargs):
# Prove these tests provide real readable source to the driver boundary.
assert any(p.read_bytes() for p in root.iterdir() if p.is_file())
assert any(p.read_bytes() for p in root.rglob("*") if p.is_file())
if self.fail:
return None
return SimpleNamespace(cpg_path=Path("test.cpg"), run_batch=self.batch, cleanup=lambda: None)
@@ -162,7 +162,7 @@ def test_later_timeout_retains_raw_answer_without_claiming_arbitration(tmp_path)
def batch(self, kind, requests):
rows = super().batch(kind, requests)
if kind == "taint" and not any(p.get("evidenceOnly") for p in requests.values()):
time.sleep(0.08)
time.sleep(0.3)
return {rid: [{"kind": "retained", "value": 42}] for rid in requests}
return rows
@@ -171,7 +171,7 @@ def test_later_timeout_retains_raw_answer_without_claiming_arbitration(tmp_path)
[region("a.py", families=("injection", "access_control"))],
backend=Slow(),
ranking_mode="static",
execution_budget=ExecutionBudget(time.monotonic() + 0.05, 0.2),
execution_budget=ExecutionBudget(time.monotonic() + 0.2, 0.2),
)
outcomes = {q.identity.family: q for q in result.question_outcomes}
assert outcomes["injection"].status == "not_arbitrated"