feat(facts): the [[layout]] fact -- vendored trees are out, test paths are not product

Two decisions the maintainer made, as rows rather than constants. A vendored tree (vendor/, node_modules/,
third_party/) is somebody else's unit: out of the targets in preprocess and out of the graph through the
frontend's --exclude, which the scan hands to the backend because the CPG seam imports only the standard
library. A test path (tests/, test-*.php, *Test.php) stays in scope -- a test that reaches a sink still
names the sink -- but its regions are not shipped and order after the product's. Measured before the row:
207 of WP Statistics's 357 PHP files were vendored, and 14 of the 67 regions ahead of MW WP Form's CVE
were test files. The position benchmark applies both, so it measures the order the scan spends.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

benchmarks/ranking/position.py

@@ -38,7 +38,9 @@ def ordered_regions(root: Path) -> list[ScanRegion]:
     regions = regions_for(analyze_entry_points(root, targets), targets, shipped=declared_sources(root))
     # The scan's own total order. If this drifts from scan.py the baseline measures the wrong thing, which
     # is why the sort key is the one thing here worth a test against the driver.
-    return sorted(regions, key=lambda r: (not r.shipped, -r.rank, r.path, r.function or ""))
+    from openultrasast.model.layout import with_layout
+
+    return sorted(with_layout(regions), key=lambda r: (not r.shipped, -r.rank, r.path, r.function or ""))
 
 
 def evidence_ordered_regions(root: Path, regions: list[ScanRegion], language: str) -> tuple[list[ScanRegion], dict[str, object]]:
@@ -58,7 +60,9 @@ def evidence_ordered_regions(root: Path, regions: list[ScanRegion], language: st
     started = time.monotonic()
     # The RECIPE's language, never the first region's: WP Statistics's first region is JavaScript, and a
     # build asked for as JavaScript went looking for jssrc2cpg and lost the whole repository's measurement.
-    cpg = backend.build(root, language=language)
+    from openultrasast.model.layout import vendored_directories
+
+    cpg = backend.build(root, language=language, exclude=vendored_directories(root))
     build_seconds = round(time.monotonic() - started, 1)
     if cpg is None:
         raise SystemExit(f"no graph for {root}: {getattr(backend, 'last_failure', '')}")

src/openultrasast/cpg/backend.py

@@ -278,7 +278,7 @@ class JoernBackend:
 
         return has_cpg()
 
-    def build(self, root: Path, *, language: str = "") -> CpgResult | None:
+    def build(self, root: Path, *, language: str = "", exclude: Sequence[str] = ()) -> CpgResult | None:
         """Build a CPG for ``root``. ``None`` on any failure — the caller degrades, never guesses.
 
         ``language`` lets a failed ``joern-parse`` retry through the frontend directly, which is what Joern
@@ -312,7 +312,7 @@ class JoernBackend:
         # A frontend whose failures are visible and retryable is worth more than one whose are not.
         if language.lower() in _PREFER_FRONTEND:
             self.last_failure = ""
-            shards, unparsed = self._build_sharded(root, scratch, language)
+            shards, unparsed = self._build_sharded(root, scratch, language, exclude=exclude)
             if not shards and self.last_failure:
                 # A build that FAILED BY NAME is not one to retry through the other launcher: the graph the
                 # census could not load would be rebuilt the same and never asked again.
@@ -450,7 +450,9 @@ class JoernBackend:
                     )
         return ""
 
-    def _build_sharded(self, root: Path, scratch: Path, language: str) -> tuple[tuple[Path, ...], tuple[str, ...]]:
+    def _build_sharded(
+        self, root: Path, scratch: Path, language: str, exclude: Sequence[str] = ()
+    ) -> tuple[tuple[Path, ...], tuple[str, ...]]:
         """Build the tree, and give the files it refuses a CPG of their OWN rather than losing them.
 
         A frontend that drops a file drops it from the only graph there is, and every question about that
@@ -465,7 +467,9 @@ class JoernBackend:
         Returns the CPGs to query and the files that defeated even a shard of their own.
         """
         # Excluded from the start, because one of them costs the entire graph rather than itself.
-        defective = self._files_with_frontend_defect(root, language)
+        # `exclude` is the caller's: the vendored trees the `[[layout]]` facts put out of scope. This seam
+        # imports only the standard library, so it is handed the names rather than reading the table.
+        defective = (*exclude, *self._files_with_frontend_defect(root, language))
         if defective:
             logger.warning(
                 "excluding %d file(s) under %s that php2cpg miscompiles (a `global` inside a closure): %s",

src/openultrasast/model/layout.py

@@ -0,0 +1,68 @@
+"""Where the `[[layout]]` facts meet a repository: which paths are vendored, which are tests.
+
+One place, imported by the preprocessor (vendored trees leave the targets), the scan (test regions are
+not product, and vendored directories leave the graph) and the position benchmark (so it measures the
+order the scan spends). Facts are rows; this is only the matching.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from collections.abc import Iterable, Sequence
+from pathlib import Path
+
+from ..semantic.facts import LayoutFact, load_facts
+from .regions import ScanRegion
+
+
+def layout_facts(language: str | None = None) -> tuple[LayoutFact, ...]:
+    """The layout rows -- for one language, or for every language when the question is about a tree."""
+    facts = load_facts()
+    return tuple(facts.for_language(language).layouts if language else facts.layouts)
+
+
+def _matches(relative: str, patterns: Iterable[str]) -> bool:
+    """gitignore-style: `dir/` names a directory at any depth, anything else a path or basename glob."""
+    parts = relative.split("/")
+    for pattern in patterns:
+        if pattern.endswith("/"):
+            if pattern.rstrip("/") in parts[:-1] or pattern.rstrip("/") in parts:
+                return True
+        elif fnmatch.fnmatch(relative, pattern) or fnmatch.fnmatch(parts[-1], pattern):
+            return True
+    return False
+
+
+def is_vendored(relative: str, facts: Sequence[LayoutFact] | None = None) -> bool:
+    rows = facts if facts is not None else layout_facts()
+    return _matches(relative, (p for fact in rows for p in fact.vendored))
+
+
+def is_test_path(relative: str, facts: Sequence[LayoutFact] | None = None) -> bool:
+    rows = facts if facts is not None else layout_facts()
+    return _matches(relative, (p for fact in rows for p in fact.tests))
+
+
+def vendored_directories(root: Path, facts: Sequence[LayoutFact] | None = None) -> tuple[str, ...]:
+    """The vendored directories actually present under ``root``, repository-relative, shallowest first --
+    what a frontend's ``--exclude`` takes."""
+    rows = facts if facts is not None else layout_facts()
+    names = {p.rstrip("/") for fact in rows for p in fact.vendored if p.endswith("/")}
+    found: list[str] = []
+    for path in sorted(root.rglob("*")):
+        if not path.is_dir() or path.name not in names:
+            continue
+        relative = path.relative_to(root).as_posix()
+        if any(relative.startswith(f"{seen}/") for seen in found):
+            continue  # already covered by a shallower vendored directory
+        found.append(relative)
+    return tuple(found)
+
+
+def with_layout(regions: Sequence[ScanRegion], facts: Sequence[LayoutFact] | None = None) -> tuple[ScanRegion, ...]:
+    """Test-path regions marked not shipped -- ordered after the product's, never removed. A region a
+    build manifest already declared shipped stays so: the manifest is the project's own word."""
+    from dataclasses import replace
+
+    rows = facts if facts is not None else layout_facts()
+    return tuple(replace(r, shipped=False) if r.shipped and is_test_path(r.path, rows) else r for r in regions)

src/openultrasast/model/scan.py

@@ -31,6 +31,7 @@ from .config_value import request_params as config_params
 from .dominance import request_params as dominance_params
 from .evidence import TIER_EXCLUDE, Evidence, evidence_from_rows
 from .ladder import Rung
+from .layout import with_layout
 from .pipeline import ModelFinding, scan_region
 from .regions import MODULE_SCOPE, ScanRegion
 from .specs import ConfigSpec, DominanceSpec, TaintSpec, config_specs, dominance_specs, taint_specs
@@ -168,7 +169,7 @@ def scan_repository(
 
     # Sort here rather than trusting the caller. The budget decides what goes unexamined, so the order it is
     # spent in belongs to whoever holds the budget.
-    ordered_regions = sorted(regions, key=lambda r: (not r.shipped, -r.rank, r.path, r.function or ""))
+    ordered_regions = sorted(with_layout(regions), key=lambda r: (not r.shipped, -r.rank, r.path, r.function or ""))
 
     # The hook link table, built once for the whole scan. It is a property of the repository rather than of
     # any region, and it is read from the source text because the graph does not carry it: php2cpg drops a
@@ -441,11 +442,17 @@ def _dispose(cpg: object) -> None:
 
 
 def _build(backend: Any, root: Path, language: str) -> Any:
-    """Build through the backend, passing the language when the backend can use it."""
+    """Build through the backend, passing the language and the vendored trees when the backend can use them."""
+    from .layout import vendored_directories
+
+    excluded = vendored_directories(root) if root.is_dir() else ()
     try:
-        return backend.build(root, language=language)
+        return backend.build(root, language=language, exclude=excluded)
     except TypeError:  # a backend from before the retry existed, including every test double
-        return backend.build(root)
+        try:
+            return backend.build(root, language=language)
+        except TypeError:
+            return backend.build(root)
 
 
 def _prefetched(rows: list[object]):  # type: ignore[no-untyped-def]

src/openultrasast/preprocess.py

@@ -97,7 +97,7 @@ def enumerate_source_files(root: Path) -> list[Path]:
     patterns = _load_ignore_patterns(root)
     paths: list[Path] = []
     for path in root.rglob("*"):
-        if not path.is_file() or _has_ignored_dir(root, path) or _is_ignored(root, path, patterns):
+        if not path.is_file() or _has_ignored_dir(root, path) or _is_ignored(root, path, patterns) or _is_vendored(root, path):
             continue
         if detect_language(path) != "unknown":
             paths.append(path)
@@ -177,6 +177,15 @@ def _has_ignored_dir(root: Path, path: Path) -> bool:
     return any(part in IGNORED_DIRS for part in relative_parts[:-1])
 
 
+def _is_vendored(root: Path, path: Path) -> bool:
+    """A tree the `[[layout]]` facts call somebody else's code. Out of the targets entirely: a bundled
+    library is a separate unit, analysed as one or not at all (contributor-scan 5.13; the decision is the
+    maintainer's and lives in the fact table, not here)."""
+    from .model.layout import is_vendored
+
+    return is_vendored(path.relative_to(root).as_posix())
+
+
 def _is_ignored(root: Path, path: Path, patterns: list[str]) -> bool:
     relative = path.relative_to(root).as_posix()
     return any(fnmatch.fnmatch(relative, pattern) or fnmatch.fnmatch(path.name, pattern) for pattern in patterns)

src/openultrasast/ruleset/semantic/php.toml

@@ -200,3 +200,13 @@ parameterized = true
 id = "wordpress_hooks"
 register = ["add_action", "add_filter", "add_shortcode"]
 apply = ["apply_filters", "apply_filters_ref_array", "do_action", "do_action_ref_array"]
+
+# What a PHP repository's paths say about scope. `vendored` trees are out of the scan and out of the graph
+# (a bundled library is a separate unit); `tests` paths stay in scope but are not product, so their
+# regions are ordered after the product's. Measured before this row: 207 of WP Statistics's 357 PHP files
+# were vendored and two of them were 60% of its graph; 14 of the 67 regions ahead of MW WP Form's CVE
+# were test-mail.php and test-mail-parser.php.
+[[layout]]
+id = "php_composer_and_wordpress"
+vendored = ["vendor/", "node_modules/", "third_party/", "third-party/", "lib/vendor/"]
+tests = ["tests/", "test/", "spec/", "test-*.php", "*Test.php", "*.test.php", "*_test.php", "phpunit/"]

src/openultrasast/semantic/facts.py

@@ -74,6 +74,27 @@ class DispatchFact:
     language: str
 
 
+@dataclass(frozen=True)
+class LayoutFact:
+    """What a repository's PATHS say about scope, per ecosystem: which trees are somebody else's code and
+    which files are tests.
+
+    Both are policy, which is why they are rows and not constants. Vendored code is a separate unit --
+    "divide and rule": a library is analysed as a library, or not at all, never as part of the project
+    that bundles it -- so a `vendored` tree is out of the targets AND out of the graph. A `tests` path is
+    in scope but not product: its regions are ordered after the product's, never removed, because a test
+    that reaches a sink still names the sink.
+
+    Patterns are gitignore-style, matched against the repository-relative path and against each directory
+    on it: `vendor/` names any directory called vendor at any depth, `test-*.php` any file so named.
+    """
+
+    id: str
+    vendored: tuple[str, ...]
+    tests: tuple[str, ...]
+    language: str
+
+
 @dataclass(frozen=True)
 class SemanticFacts:
     version: str
@@ -81,6 +102,7 @@ class SemanticFacts:
     sinks: tuple[SinkFact, ...]
     sanitizers: tuple[SanitizerFact, ...]
     dispatches: tuple[DispatchFact, ...] = ()
+    layouts: tuple[LayoutFact, ...] = ()
 
     def for_language(self, language: str) -> SemanticFacts:
         key = _LANGUAGE_ALIASES.get(language, language)
@@ -90,6 +112,7 @@ class SemanticFacts:
             sinks=tuple(item for item in self.sinks if item.language == key),
             sanitizers=tuple(item for item in self.sanitizers if item.language == key),
             dispatches=tuple(item for item in self.dispatches if item.language == key),
+            layouts=tuple(item for item in self.layouts if item.language == key),
         )
 
 
@@ -101,6 +124,7 @@ def load_facts(directory: Path | None = None) -> SemanticFacts:
     sinks: list[SinkFact] = []
     sanitizers: list[SanitizerFact] = []
     dispatches: list[DispatchFact] = []
+    layouts: list[LayoutFact] = []
     versions: list[str] = []
     found = False
     for path in sorted(root.glob("*.toml")):
@@ -115,6 +139,7 @@ def load_facts(directory: Path | None = None) -> SemanticFacts:
         sinks.extend(_sinks(payload.get("sink"), language, path))
         sanitizers.extend(_sanitizers(payload.get("sanitizer"), language, path))
         dispatches.extend(_dispatches(payload.get("dispatch"), language, path))
+        layouts.extend(_layouts(payload.get("layout"), language, path))
     if not found:
         raise FactLoadError(f"no semantic fact files in {root}")
     return SemanticFacts(
@@ -123,6 +148,7 @@ def load_facts(directory: Path | None = None) -> SemanticFacts:
         sinks=tuple(sinks),
         sanitizers=tuple(sanitizers),
         dispatches=tuple(dispatches),
+        layouts=tuple(layouts),
     )
 
 
@@ -143,6 +169,23 @@ def _dispatches(value: object, language: str, path: Path) -> list[DispatchFact]:
     return facts
 
 
+def _layouts(value: object, language: str, path: Path) -> list[LayoutFact]:
+    facts: list[LayoutFact] = []
+    for item in _items(value):
+        identifier = str(item.get("id", "")).strip()
+        if not identifier:
+            raise FactLoadError(f"layout without an id in {path}")
+        facts.append(
+            LayoutFact(
+                id=identifier,
+                vendored=_strings(item.get("vendored", []), "vendored", path),
+                tests=_strings(item.get("tests", []), "tests", path),
+                language=language,
+            )
+        )
+    return facts
+
+
 def _items(value: object) -> list[dict[str, object]]:
     if value is None:
         return []

tests/test_layout_facts.py

@@ -0,0 +1,87 @@
+"""The `[[layout]]` facts: which paths are somebody else's code, which are tests -- rows, not constants."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+
+def test_layout_rows_load_and_match() -> None:
+    from openultrasast.model.layout import is_test_path, is_vendored, layout_facts
+
+    rows = layout_facts("php")
+    assert [r.id for r in rows] == ["php_composer_and_wordpress"]
+    assert is_vendored("includes/vendor/whichbrowser/parser/data/profiles.php", rows)
+    assert is_vendored("vendor/autoload.php", rows)
+    assert not is_vendored("classes/class.mail.php", rows)
+    assert is_test_path("test-mail.php", rows) and is_test_path("tests/unit/MailTest.php", rows)
+    assert is_test_path("classes/FooTest.php", rows)
+    assert not is_test_path("classes/class.mail.php", rows) and not is_test_path("includes/testimonials.php", rows)
+
+
+def test_a_layout_row_without_an_id_is_refused(tmp_path: Path) -> None:
+    from openultrasast.semantic.facts import FactLoadError, load_facts
+
+    (tmp_path / "php.toml").write_text('[[layout]]\nvendored = ["vendor/"]\n')
+    with pytest.raises(FactLoadError):
+        load_facts(tmp_path)
+
+
+def test_vendored_trees_leave_the_targets(tmp_path: Path) -> None:
+    """A bundled library is a separate unit -- analysed as one or not at all, never as part of the project
+    that bundles it. Measured before the row: 207 of WP Statistics's 357 PHP files were vendored."""
+    from openultrasast.preprocess import preprocess_repository
+
+    (tmp_path / "classes").mkdir()
+    (tmp_path / "classes" / "a.php").write_text("<?php\n")
+    (tmp_path / "includes" / "vendor" / "lib").mkdir(parents=True)
+    (tmp_path / "includes" / "vendor" / "lib" / "b.php").write_text("<?php\n")
+    _, targets = preprocess_repository(tmp_path)
+    assert sorted(t.path for t in targets) == ["classes/a.php"]
+
+
+def test_vendored_directories_are_named_shallowest_first(tmp_path: Path) -> None:
+    from openultrasast.model.layout import vendored_directories
+
+    (tmp_path / "includes" / "vendor" / "x" / "vendor").mkdir(parents=True)
+    (tmp_path / "vendor").mkdir()
+    (tmp_path / "src").mkdir()
+    assert vendored_directories(tmp_path) == ("includes/vendor", "vendor")
+
+
+def test_test_regions_are_not_product_and_order_last() -> None:
+    """In scope -- a test that reaches a sink still names the sink -- but after the product's regions."""
+    from openultrasast.model.layout import with_layout
+    from openultrasast.model.regions import ScanRegion
+
+    product = ScanRegion(path="classes/class.mail.php", function="_delete_files", language="php", families=("path",), rank=0.3, source="x")
+    test = ScanRegion(path="test-mail.php", function="hook_mwform_admin_mail", language="php", families=("path",), rank=0.3, source="x")
+    declared = ScanRegion(path="tests/fixture.php", function="f", language="php", families=("path",), rank=0.3, source="x", shipped=True)
+    out = {r.path: r.shipped for r in with_layout([product, test])}
+    assert out == {"classes/class.mail.php": True, "test-mail.php": False}
+    assert with_layout([declared])[0].shipped is False, (
+        "a test path is a test path; only a manifest that lists it as a source would say otherwise, and it is applied first"
+    )
+
+
+def test_the_scan_hands_the_vendored_trees_to_the_build(tmp_path: Path) -> None:
+    from openultrasast.model.regions import ScanRegion
+    from openultrasast.model.scan import ScanBudget, scan_repository
+
+    (tmp_path / "vendor").mkdir()
+    seen: dict[str, object] = {}
+
+    class _Backend:
+        def available(self) -> bool:
+            return True
+
+        def build(self, root, *, language="", exclude=()):  # type: ignore[no-untyped-def]
+            from openultrasast.cpg.backend import CpgResult
+
+            seen["exclude"] = tuple(exclude)
+            return CpgResult(cpg_path=Path("cpg.bin"), run=lambda q, p: [], run_batch=lambda q, r: {rid: [] for rid in r})
+
+    region = ScanRegion(path="a.php", function="f", language="php", families=("injection",), rank=0.5, source="x")
+    scan_repository(tmp_path, [region], backend=_Backend(), budget=ScanBudget(max_model_calls=0))
+    assert seen["exclude"] == ("vendor",)