Open_Ultra_SAST/src/openultrasast/semantic/facts.py

"""Source, sink, and sanitizer facts as versioned data next to the ruleset.

An entry that knows a framework or a library carries ``framework = "<id>"`` or ``library = "<id>"`` (an id of
``ruleset/frameworks.toml``): a prior (learned-decision-engine Req 8.2). :func:`load_facts` keeps or drops the tagged
entries by ``priors`` -- ``"all"`` (the default: today's facts), ``"off"`` (language-level only) or a set of ids.
"""

from __future__ import annotations

import tomllib
from dataclasses import dataclass
from pathlib import Path

from ..ruleset.frameworks import Priors, kept, normalize_priors, read_tag, tag_of

DEFAULT_FACTS_DIR = Path(__file__).resolve().parents[1] / "ruleset" / "semantic"

TAINT_LANGUAGE_ALIAS = {"typescript": "javascript"}

_LANGUAGE_ALIASES = {
    "c": "c",
    "cpp": "c",
    "python": "python",
    "javascript": "javascript",
    **TAINT_LANGUAGE_ALIAS,
    "java": "java",
}


class FactLoadError(ValueError):
    """Raised when semantic fact files are missing or invalid."""

    def __init__(self, message: str, reason: str = "facts_unavailable") -> None:
        super().__init__(message)
        self.reason = reason


@dataclass(frozen=True)
class SourceFact:
    id: str
    patterns: tuple[str, ...]
    language: str
    framework: str | None = None
    library: str | None = None


@dataclass(frozen=True)
class SinkFact:
    id: str
    cwe: str
    calls: tuple[str, ...]
    rule_ids: tuple[str, ...]
    language: str
    weak_literals: tuple[str, ...] = ()
    format_arg: int | None = None
    # A DESTINATION sink (a redirect, an outbound request) is dangerous only where the value can choose where
    # it goes. With `prefix_fixes_origin`, a value appended after a prefix that already fixes the origin -- a
    # path `/x`, a `?query`, `scheme://host/`, or an `origin_anchors` call such as Django's `reverse()` -- is
    # not an untrusted destination, and neither is one that only fills an anchor call's arguments.
    prefix_fixes_origin: bool = False
    origin_anchors: tuple[str, ...] = ()
    framework: str | None = None
    library: str | None = None


@dataclass(frozen=True)
class SanitizerFact:
    id: str
    calls: tuple[str, ...]
    language: str
    parameterized: bool = False
    literal_format_arg: int | None = None
    # An ESCAPE, not a cleansing: `esc_sql` backslashes quotes, which protects a value inside a quoted SQL
    # literal and nothing else. `ORDER BY u.{$sortby}` is Ultimate Member's CVE-2024-1071, escaped and injectable.
    quoted_only: bool = False
    # A CHECK, not a transformation: `in_array($v, $allowed, true)`, `url_has_allowed_host_and_scheme(v)`. It
    # makes a value safe only where it held -- inside the branch it guards, past an exit on its failure, or in
    # the arm of a ternary it selects. Never a cleansing call on a path.
    guard: bool = False
    framework: str | None = None
    library: str | None = None


@dataclass(frozen=True)
class DispatchFact:
    """A framework that connects a producer to a consumer through a STRING key rather than a call.

    WordPress is the example that forced this, but nothing here is about WordPress. The shape is general --
    a registry keyed by a name, written as a literal at both ends -- and it is how Django signals, jQuery
    events, Symfony's event dispatcher and every plugin system of this kind route data. No CPG frontend can
    draw the edge, because neither end names the other.

    ``register`` names the calls that put a callback INTO the registry, ``apply`` the calls that take a value
    back OUT of it. Both belong in a fact table for the same reason sinks do: a query that hardcodes
    `apply_filters` is a query that only works on one framework, and the next one needs a code change rather
    than a row.
    """

    id: str
    register: tuple[str, ...]
    apply: tuple[str, ...]
    language: str
    # What an apply call RETURNS. `returns` names the apply calls whose result is the dispatched value, and
    # `value_arg` is that value's argument position (1-based, receiver excluded): WordPress's
    # `apply_filters('hook', $value, ...$context)` returns the filtered `$value`, and its context arguments
    # never become the result. An apply call not named in `returns` returns nothing a flow can carry
    # (`do_action`). Without this the engine let EVERY argument reach the return: PMPro's random order code
    # came out "tainted" because `$this` was passed as filter context. Empty keeps the engine's default.
    returns: tuple[str, ...] = ()
    value_arg: int = 0
    # The `register` calls whose callbacks are handed REQUEST-BORNE arguments. Every other registration's
    # callback receives what the matching apply call passes -- framework or plugin values such as the
    # `WP_User` a profile hook gets -- so its parameters are not attacker input; where request data does
    # reach an apply call's arguments, the hook half of the taint query follows it from there.
    request_arguments: tuple[str, ...] = ()
    framework: str | None = None
    library: str | None = None


@dataclass(frozen=True)
class LayoutFact:
    """What a repository's PATHS say about scope, per ecosystem: which trees are somebody else's code and
    which files are tests.

    Both are policy, which is why they are rows and not constants. Vendored code is a separate unit --
    "divide and rule": a library is analysed as a library, or not at all, never as part of the project
    that bundles it -- so a `vendored` tree is out of the targets AND out of the graph. A `tests` path is
    in scope but not product: its regions are ordered after the product's, never removed, because a test
    that reaches a sink still names the sink.

    Patterns are gitignore-style, matched against the repository-relative path and against each directory
    on it: `vendor/` names any directory called vendor at any depth, `test-*.php` any file so named.
    """

    id: str
    vendored: tuple[str, ...]
    tests: tuple[str, ...]
    language: str


@dataclass(frozen=True)
class SemanticFacts:
    version: str
    sources: tuple[SourceFact, ...]
    sinks: tuple[SinkFact, ...]
    sanitizers: tuple[SanitizerFact, ...]
    dispatches: tuple[DispatchFact, ...] = ()
    layouts: tuple[LayoutFact, ...] = ()

    def taint_language(self, language: str) -> str:
        """Prefer native facts when supplied; otherwise use the shared language aliases."""
        if language in TAINT_LANGUAGE_ALIAS and any(
            item.language == language
            for items in (self.sources, self.sinks, self.sanitizers, self.dispatches, self.layouts)
            for item in items
        ):
            return language
        return _LANGUAGE_ALIASES.get(language, language)

    def for_language(self, language: str) -> SemanticFacts:
        key = self.taint_language(language)
        return SemanticFacts(
            version=self.version,
            sources=tuple(item for item in self.sources if item.language == key),
            sinks=tuple(item for item in self.sinks if item.language == key),
            sanitizers=tuple(item for item in self.sanitizers if item.language == key),
            dispatches=tuple(item for item in self.dispatches if item.language == key),
            layouts=tuple(item for item in self.layouts if item.language == key),
        )


def _tags(item: dict[str, object], path: Path) -> dict[str, str | None]:
    framework, library = read_tag(item, f"{path} {item.get('id')!r}")
    return {"framework": framework, "library": library}


def with_priors(facts: SemanticFacts, priors: Priors) -> SemanticFacts:
    """``facts`` with the tagged entries ``priors`` does not keep removed (language-level entries always stay)."""
    if priors == "all":
        return facts
    return SemanticFacts(
        version=facts.version,
        sources=tuple(f for f in facts.sources if kept(tag_of(f), priors)),
        sinks=tuple(f for f in facts.sinks if kept(tag_of(f), priors)),
        sanitizers=tuple(f for f in facts.sanitizers if kept(tag_of(f), priors)),
        dispatches=tuple(f for f in facts.dispatches if kept(tag_of(f), priors)),
        layouts=facts.layouts,
    )


def _load_facts_uncached(directory: Path | None = None) -> SemanticFacts:
    root = directory if directory is not None else DEFAULT_FACTS_DIR
    if not root.is_dir():
        raise FactLoadError(f"semantic fact directory missing: {root}")
    sources: list[SourceFact] = []
    sinks: list[SinkFact] = []
    sanitizers: list[SanitizerFact] = []
    dispatches: list[DispatchFact] = []
    layouts: list[LayoutFact] = []
    versions: list[str] = []
    found = False
    for path in sorted(root.glob("*.toml")):
        found = True
        try:
            payload = tomllib.loads(path.read_text())
        except (OSError, tomllib.TOMLDecodeError) as exc:
            raise FactLoadError(f"invalid semantic fact file {path}: {exc}") from exc
        language = str(payload.get("language") or path.stem)
        versions.append(str(payload.get("version", "1")))
        sources.extend(_sources(payload.get("source"), language, path))
        sinks.extend(_sinks(payload.get("sink"), language, path))
        sanitizers.extend(_sanitizers(payload.get("sanitizer"), language, path))
        dispatches.extend(_dispatches(payload.get("dispatch"), language, path))
        layouts.extend(_layouts(payload.get("layout"), language, path))
    if not found:
        raise FactLoadError(f"no semantic fact files in {root}")
    return SemanticFacts(
        version=versions[0] if versions else "1",
        sources=tuple(sources),
        sinks=tuple(sinks),
        sanitizers=tuple(sanitizers),
        dispatches=tuple(dispatches),
        layouts=tuple(layouts),
    )


def _dispatches(value: object, language: str, path: Path) -> list[DispatchFact]:
    facts: list[DispatchFact] = []
    for item in _items(value):
        identifier = str(item.get("id", "")).strip()
        if not identifier:
            raise FactLoadError(f"dispatch without an id in {path}")
        facts.append(
            DispatchFact(
                id=identifier,
                register=_strings(item.get("register"), "register", path),
                apply=_strings(item.get("apply"), "apply", path),
                language=language,
                returns=_returning(item, path),
                value_arg=_value_arg(item, path),
                request_arguments=_request_arguments(item, path),
                **_tags(item, path),
            )
        )
    return facts


def _returning(item: dict[str, object], path: Path) -> tuple[str, ...]:
    returns = _strings(item.get("returns", []), "returns", path)
    unknown = set(returns) - set(_strings(item.get("apply"), "apply", path))
    if unknown:
        raise FactLoadError(f"dispatch {item.get('id')!r} in {path}: returns names calls that are not apply calls: {sorted(unknown)}")
    return returns


def _request_arguments(item: dict[str, object], path: Path) -> tuple[str, ...]:
    calls = _strings(item.get("request_arguments", []), "request_arguments", path)
    unknown = set(calls) - set(_strings(item.get("register"), "register", path))
    if unknown:
        raise FactLoadError(
            f"dispatch {item.get('id')!r} in {path}: request_arguments names calls that are not register calls: {sorted(unknown)}"
        )
    return calls


def _value_arg(item: dict[str, object], path: Path) -> int:
    value = item.get("value_arg", 0)
    if not isinstance(value, int) or isinstance(value, bool) or value < 0:
        raise FactLoadError(f"dispatch {item.get('id')!r} in {path}: value_arg must be a non-negative integer")
    if item.get("returns") and value < 1:
        raise FactLoadError(f"dispatch {item.get('id')!r} in {path}: returns needs the value's argument position")
    return value


def _layouts(value: object, language: str, path: Path) -> list[LayoutFact]:
    facts: list[LayoutFact] = []
    for item in _items(value):
        identifier = str(item.get("id", "")).strip()
        if not identifier:
            raise FactLoadError(f"layout without an id in {path}")
        facts.append(
            LayoutFact(
                id=identifier,
                vendored=_strings(item.get("vendored", []), "vendored", path),
                tests=_strings(item.get("tests", []), "tests", path),
                language=language,
            )
        )
    return facts


def _items(value: object) -> list[dict[str, object]]:
    if value is None:
        return []
    if not isinstance(value, list):
        raise FactLoadError("fact tables must be arrays")
    rows: list[dict[str, object]] = []
    for item in value:
        if not isinstance(item, dict):
            raise FactLoadError("each fact row must be a table")
        rows.append(item)
    return rows


def _strings(value: object, field: str, path: Path) -> tuple[str, ...]:
    if value is None:
        return ()
    if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value):
        raise FactLoadError(f"{path} has invalid {field} list")
    return tuple(str(item) for item in value)


def _sources(value: object, language: str, path: Path) -> list[SourceFact]:
    rows: list[SourceFact] = []
    for item in _items(value):
        ident = item.get("id")
        if not isinstance(ident, str) or not ident:
            raise FactLoadError(f"{path} source is missing id")
        patterns = _strings(item.get("patterns"), "patterns", path)
        if not patterns:
            raise FactLoadError(f"{path} source {ident} has no patterns")
        rows.append(SourceFact(id=ident, patterns=patterns, language=language, **_tags(item, path)))
    return rows


def _flag(value: object, field: str, path: Path, ident: str) -> bool:
    if value is None:
        return False
    if not isinstance(value, bool):
        raise FactLoadError(f"{path} fact {ident} has a non-boolean {field}")
    return value


def _sinks(value: object, language: str, path: Path) -> list[SinkFact]:
    rows: list[SinkFact] = []
    for item in _items(value):
        ident = item.get("id")
        cwe = item.get("cwe")
        if not isinstance(ident, str) or not ident:
            raise FactLoadError(f"{path} sink is missing id")
        if not isinstance(cwe, str) or not cwe.startswith("CWE-"):
            raise FactLoadError(f"{path} sink {ident} must bind a CWE id")
        calls = _strings(item.get("calls"), "calls", path)
        if not calls:
            raise FactLoadError(f"{path} sink {ident} has no calls")
        format_arg = item.get("format_arg")
        if format_arg is not None and (not isinstance(format_arg, int) or isinstance(format_arg, bool)):
            raise FactLoadError(f"{path} sink {ident} has invalid format_arg")
        rows.append(
            SinkFact(
                id=ident,
                cwe=cwe,
                calls=calls,
                rule_ids=_strings(item.get("rule_ids"), "rule_ids", path),
                language=language,
                weak_literals=_strings(item.get("weak_literals"), "weak_literals", path),
                format_arg=format_arg if isinstance(format_arg, int) else None,
                prefix_fixes_origin=_flag(item.get("prefix_fixes_origin"), "prefix_fixes_origin", path, ident),
                origin_anchors=_strings(item.get("origin_anchors"), "origin_anchors", path),
                **_tags(item, path),
            )
        )
    return rows


def _sanitizers(value: object, language: str, path: Path) -> list[SanitizerFact]:
    rows: list[SanitizerFact] = []
    for item in _items(value):
        ident = item.get("id")
        if not isinstance(ident, str) or not ident:
            raise FactLoadError(f"{path} sanitizer is missing id")
        calls = _strings(item.get("calls"), "calls", path)
        if not calls:
            raise FactLoadError(f"{path} sanitizer {ident} has no calls")
        literal = item.get("literal_format_arg")
        if literal is not None and (not isinstance(literal, int) or isinstance(literal, bool)):
            raise FactLoadError(f"{path} sanitizer {ident} has invalid literal_format_arg")
        rows.append(
            SanitizerFact(
                id=ident,
                calls=calls,
                language=language,
                parameterized=bool(item.get("parameterized", False)),
                literal_format_arg=literal if isinstance(literal, int) else None,
                quoted_only=_flag(item.get("quoted_only"), "quoted_only", path, ident),
                guard=_flag(item.get("guard"), "guard", path, ident),
                **_tags(item, path),
            )
        )
    return rows


# Parsed once per ruleset STATE, not once per call. Region discovery asks for the family specs of every region,
# and each spec call loaded the facts again: 89,127 TOML parses and 275 s of a 339 s discovery on a WordPress
# plugin, paid twice per push. The key is every fact file's name, size and modification time, so an edited
# ruleset -- a test's fixture, a maintainer's change -- is read again, and the result is immutable, so sharing
# it is safe. A load that fails is not remembered.
_FACTS_MEMO: dict[tuple[object, ...], SemanticFacts] = {}


_LISTED: dict[tuple[str, str], tuple[int, tuple[Path, ...]]] = {}


def _ruleset_state(root: Path, pattern: str) -> tuple[object, ...]:
    """Every matching file's name, size and modification time -- cheaply, because every question asks.

    The directory is listed again only when its own stamp moves (a file added, removed or renamed); an edit in
    place moves the file's stamp, which is read every time. Globbing and resolving on every call was 7% of a
    whole push with the engine stubbed out.
    """
    stamp = root.stat().st_mtime_ns
    listed = _LISTED.get((str(root), pattern))
    if listed is None or listed[0] != stamp:
        listed = (stamp, tuple(sorted(root.glob(pattern))))
        _LISTED[(str(root), pattern)] = listed
    return (str(root), stamp, *((path.name, *_stat_of(path)) for path in listed[1]))


def _stat_of(path: Path) -> tuple[int, int]:
    info = path.stat()
    return info.st_size, info.st_mtime_ns


def load_facts(directory: Path | None = None, *, priors: Priors = "all") -> SemanticFacts:
    """The facts of ``directory`` (default: the bundled ruleset), with the priors ``priors`` keeps: ``"all"`` (the
    default, today's facts), ``"off"`` or a set of framework/library ids."""
    chosen = normalize_priors(priors)
    root = directory if directory is not None else DEFAULT_FACTS_DIR
    if not root.is_dir():
        return with_priors(_load_facts_uncached(directory), chosen)
    key = _ruleset_state(root, "*.toml")
    if key not in _FACTS_MEMO:
        _FACTS_MEMO[key] = _load_facts_uncached(directory)
    if chosen == "all":
        return _FACTS_MEMO[key]
    scoped = (*key, "priors", chosen if isinstance(chosen, str) else tuple(sorted(chosen)))
    if scoped not in _FACTS_MEMO:
        _FACTS_MEMO[scoped] = with_priors(_FACTS_MEMO[key], chosen)
    return _FACTS_MEMO[scoped]