From af81bccc70e09343e1ca0ae9151efcd803f61b83 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:49:05 +0200 Subject: [PATCH] feat(ruleset): tag framework-specific knowledge as optional priors (decision-engine task 2) ruleset/frameworks.toml lists 19 framework and library ids (wordpress, flask, django, express, spring, jakarta-servlet, react, vue and 11 libraries), each with its languages, dependency package names and a symbol lexicon used only by the lint and for stratification. Every framework/library entry of section 2 is tagged `framework`/`library`, and mixed entries are split so a tag covers a whole entry (split pieces keep the entry's id and flags, in place): semantic facts 38 tagged entries (wordpress 6, django 6, flask 4, express 4, jakarta-servlet 3, pyyaml 2, one each for spring, sqlalchemy, requests, httpx, lodash, node-http-clients, escape-html, sanitize-html, dompurify, esapi, commons-text, react, vue); the dispatch fact behind the hook-as-source path is wordpress; the PHP layout row is renamed php_layout and stays untagged (a product filter). Quick rules: 9 tagged (wordpress 6, flask, pyyaml, node-http-clients); php-unserialize is split into php-unserialize and php-maybe-unserialize; the lint also caught WordPress escapers in exclusions and axios/yaml.load in mixed rules, so those keep their ids as tagged rules and gain a `fallback_for` language-level rule (6) that is active only when the prior is off. load_facts and load_ruleset take priors ("all" by default -- today's scan --, "off", or a set of ids). tests/test_semantic_priors.py: the lint (no untagged entry contains a lexicon symbol of its language), "off" removes exactly the tagged entries and activates the fallbacks, split entries keep their language-level calls. Rule and layout counts in three existing tests follow the split and the rename. Unchanged defaults: test_benchmark_output_is_byte_identical_to_golden_baseline passes, and the quick findings of all 15 benchmark manifests (full StaticFinding records) are byte-identical before and after. priors=off on the same manifests: 61 -> 57 findings; the 4 lost are js-ssrf (axios), php-wpdb-sql-composition, python-ssti and python yaml.load; 5 PHP and 1 Python finding move to their fallback rule at the same location. Semantic calls with priors off: php 75/92, python 32/77, javascript 42/70, java 24/41, c 27/27 (the engine-side effect needs Joern and was not measured). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .kiro/specs/learned-decision-engine/tasks.md | 2 +- .../measurements/2026-09-08-module-audit.json | 15 +- src/openultrasast/ruleset/frameworks.py | 126 +++++++++++++ src/openultrasast/ruleset/frameworks.toml | 160 +++++++++++++++++ .../ruleset/javascript/rules.toml | 15 ++ src/openultrasast/ruleset/php/rules.toml | 77 +++++++- src/openultrasast/ruleset/python/rules.toml | 16 ++ src/openultrasast/ruleset/semantic/java.toml | 37 +++- .../ruleset/semantic/javascript.toml | 94 ++++++++-- src/openultrasast/ruleset/semantic/php.toml | 32 +++- .../ruleset/semantic/python.toml | 79 ++++++++- src/openultrasast/ruleset/store.py | 32 +++- src/openultrasast/semantic/facts.py | 55 +++++- tests/test_layout_facts.py | 2 +- tests/test_php_quick_rules.py | 3 +- tests/test_ruleset.py | 2 +- tests/test_semantic_priors.py | 166 ++++++++++++++++++ 17 files changed, 872 insertions(+), 41 deletions(-) create mode 100644 src/openultrasast/ruleset/frameworks.py create mode 100644 src/openultrasast/ruleset/frameworks.toml create mode 100644 tests/test_semantic_priors.py diff --git a/.kiro/specs/learned-decision-engine/tasks.md b/.kiro/specs/learned-decision-engine/tasks.md index 8842d68..c47ce4b 100644 --- a/.kiro/specs/learned-decision-engine/tasks.md +++ b/.kiro/specs/learned-decision-engine/tasks.md @@ -11,7 +11,7 @@ the maintainer confirms its budget. allow-list and the exclusion of label-carrying fields. - _Requirements: 1.1, 1.2, 1.3, 7.2_ -- [ ] 2. Framework tags +- [x] 2. Framework tags - `ruleset/frameworks.toml`; tag and split every framework/library entry listed in design section 2; `priors=` in the semantic and quick loaders (default `all`, so today's scan is unchanged -- the golden benchmark test stays byte-identical); a lint test that no untagged framework entry remains. diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index 8a2c0e7..c7f524a 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,7 +1,7 @@ { - "modules": 139, + "modules": 140, "counts": { - "load_bearing": 132, + "load_bearing": 133, "standalone_capability": 7 }, "orphaned": [], @@ -1199,9 +1199,20 @@ "plane.tasks.alerts", "plane.tasks.features", "plane.tasks.loop", + "ruleset.store", + "semantic.facts", "stage_processors" ] }, + { + "module": "ruleset.frameworks", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "ruleset.store", + "semantic.facts" + ] + }, { "module": "ruleset.store", "classification": "load_bearing", diff --git a/src/openultrasast/ruleset/frameworks.py b/src/openultrasast/ruleset/frameworks.py new file mode 100644 index 0000000..6a0fe28 --- /dev/null +++ b/src/openultrasast/ruleset/frameworks.py @@ -0,0 +1,126 @@ +"""Framework and library priors (learned-decision-engine Req 8.2, design section 2). + +``frameworks.toml`` lists the framework and library ids a ruleset entry may be tagged with, each with a lexicon of +symbols. A tagged entry (``framework = ""`` or ``library = ""`` on a ``semantic/*.toml`` source, sink, +sanitizer or dispatch, or on a quick ``[[rule]]``) is a *prior*: the loaders keep it or drop it by ``priors``: + +- ``"all"``: every entry (today's scan: the default of every loader, so output is unchanged); +- ``"off"``: language-level entries only (the decision engine's default); +- a set of ids: language-level entries plus those priors. + +The lexicon is for the lint test and repository stratification, never for detection. +""" + +from __future__ import annotations + +import re +import tomllib +from collections.abc import Iterable +from dataclasses import dataclass +from functools import cache +from pathlib import Path +from typing import Literal + +DEFAULT_FRAMEWORKS_PATH = Path(__file__).resolve().parent / "frameworks.toml" +KINDS = ("framework", "library") +Priors = Literal["off", "all"] | frozenset[str] + + +class FrameworksError(ValueError): + """``frameworks.toml`` or a tag is malformed, or names an id the file does not list.""" + + +@dataclass(frozen=True) +class Framework: + id: str + kind: str # KINDS + languages: tuple[str, ...] + packages: tuple[str, ...] + symbols: tuple[str, ...] + markers: tuple[str, ...] = () + + +def _strings(value: object, field: str, where: str) -> tuple[str, ...]: + if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value): + raise FrameworksError(f"{where}: {field} must be a list of non-empty strings") + return tuple(value) + + +@cache +def load_frameworks(path: Path = DEFAULT_FRAMEWORKS_PATH) -> tuple[Framework, ...]: + data = tomllib.loads(Path(path).read_text(encoding="utf-8")) + rows: list[Framework] = [] + for item in data.get("framework", []): + where = f"{path} framework {item.get('id')!r}" + if not isinstance(item.get("id"), str) or item.get("kind") not in KINDS: + raise FrameworksError(f"{where}: needs an id and a kind in {KINDS}") + extra = set(item) - {"id", "kind", "languages", "packages", "symbols", "markers"} + if extra: + raise FrameworksError(f"{where}: unknown fields {sorted(extra)}") + rows.append( + Framework( + id=item["id"], kind=item["kind"], languages=_strings(item.get("languages"), "languages", where), + packages=_strings(item.get("packages", []), "packages", where), symbols=_strings(item.get("symbols"), "symbols", where), + markers=_strings(item.get("markers", []), "markers", where), + ) + ) # fmt: skip + ids = [row.id for row in rows] + if len(ids) != len(set(ids)): + raise FrameworksError(f"{path}: an id is listed twice") + return tuple(rows) + + +def framework_ids() -> frozenset[str]: + return frozenset(row.id for row in load_frameworks()) + + +def normalize_priors(priors: Priors | Iterable[str]) -> Priors: + """``"all"``, ``"off"`` or a frozenset of known ids; an unknown id is refused, never silently ignored.""" + if priors in ("all", "off"): + return priors # type: ignore[return-value] + if isinstance(priors, str): + raise FrameworksError(f"priors must be 'all', 'off' or a set of framework ids, got {priors!r}") + chosen = frozenset(priors) + unknown = sorted(chosen - framework_ids()) + if unknown: + raise FrameworksError(f"unknown priors {unknown} (frameworks.toml lists {sorted(framework_ids())})") + return chosen + + +def tag_of(item: object) -> str | None: + """The framework or library id an entry is tagged with, or None for a language-level entry.""" + return getattr(item, "framework", None) or getattr(item, "library", None) + + +def kept(tag: str | None, priors: Priors) -> bool: + """Whether an entry with ``tag`` survives ``priors``: language-level entries always do.""" + if tag is None or priors == "all": + return True + if priors == "off": + return False + return tag in priors + + +def read_tag(item: dict[str, object], where: str) -> tuple[str | None, str | None]: + """(framework, library) of a TOML entry, checked: at most one, and a listed id of the matching kind.""" + framework, library = item.get("framework"), item.get("library") + if framework is not None and library is not None: + raise FrameworksError(f"{where}: tagged both framework and library; one tag covers a whole entry") + for value, kind in ((framework, "framework"), (library, "library")): + if value is None: + continue + rows = {row.id: row for row in load_frameworks()} + if not isinstance(value, str) or value not in rows or rows[value].kind != kind: + raise FrameworksError(f"{where}: {kind} = {value!r} is not a {kind} id of frameworks.toml") + return (framework if isinstance(framework, str) else None), (library if isinstance(library, str) else None) + + +def symbol_in(symbol: str, text: str) -> bool: + """The lexicon's match rule: ``symbol`` as a whole token of ``text``.""" + return re.search(r"(?"` or `library = ""` naming one row here, and loaders take +# `priors` ("all" -- today's scan --, "off", or a set of ids) to keep or drop the tagged entries. +# +# `symbols` is a lexicon with two uses only: the lint test (`tests/test_semantic_priors.py`) fails on an untagged +# entry that contains one, and repository stratification (with `packages`, the dependency names a manifest such as +# composer.json, requirements*.txt, pyproject.toml, package.json or pom.xml declares). It never feeds detection. +# A symbol matches as a whole token: not preceded or followed by a letter, digit or underscore. + +version = "1" + +[[framework]] +id = "wordpress" +kind = "framework" +languages = ["php"] +packages = ["johnpbloch/wordpress", "roots/wordpress", "wordpress/wordpress"] +# A plugin or theme checkout declares itself in a file header instead of a dependency manifest. +markers = ["Plugin Name:", "Theme Name:"] +symbols = [ + "$wpdb", "wpdb->", "maybe_unserialize", "esc_sql", "absint", "esc_html", "esc_attr", "esc_url", "esc_js", + "sanitize_text_field", "wp_kses", "add_action", "add_filter", "add_shortcode", "apply_filters", + "apply_filters_ref_array", "do_action", "do_action_ref_array", +] + +[[framework]] +id = "flask" +kind = "framework" +languages = ["python"] +packages = ["flask"] +symbols = [ + "request.args", "request.form", "request.json", "request.values", "request.get_json", "request.files", + "send_file", "send_from_directory", "render_template_string", "Markup", "url_for", +] + +[[framework]] +id = "django" +kind = "framework" +languages = ["python"] +packages = ["django", "djangorestframework"] +symbols = [ + "request.GET", "request.POST", "request.COOKIES", "request.FILES", "request.META", "request.query_params", + "cleaned_data", "objects.raw", "RawSQL", "mark_safe", "format_html", "HttpResponse", "HttpResponseRedirect", + "FileResponse", "url_has_allowed_host_and_scheme", "is_safe_url", "reverse_lazy", +] + +[[framework]] +id = "express" +kind = "framework" +languages = ["javascript", "typescript"] +packages = ["express"] +symbols = ["req.query", "req.body", "req.params", "req.cookies", "req.headers", "res.send", "response.send", "sendFile", "res.redirect"] + +[[framework]] +id = "spring" +kind = "framework" +languages = ["java"] +packages = ["org.springframework:spring-web", "org.springframework.boot:spring-boot-starter-web"] +symbols = ["getForObject", "postForObject", "RestTemplate", "WebClient"] + +[[framework]] +id = "jakarta-servlet" +kind = "framework" +languages = ["java"] +packages = ["jakarta.servlet:jakarta.servlet-api", "javax.servlet:javax.servlet-api"] +symbols = ["getParameter", "getQueryString", "getWriter", "sendRedirect", "HttpServletRequest", "HttpServletResponse"] + +[[framework]] +id = "react" +kind = "framework" +languages = ["javascript", "typescript"] +packages = ["react", "react-dom"] +symbols = ["dangerouslySetInnerHTML"] + +[[framework]] +id = "vue" +kind = "framework" +languages = ["javascript", "typescript"] +packages = ["vue"] +symbols = ["v-html"] + +[[framework]] +id = "sqlalchemy" +kind = "library" +languages = ["python"] +packages = ["sqlalchemy"] +symbols = ["from_statement"] + +[[framework]] +id = "pyyaml" +kind = "library" +languages = ["python"] +packages = ["pyyaml"] +symbols = ["yaml.load", "yaml.safe_load"] + +[[framework]] +id = "requests" +kind = "library" +languages = ["python"] +packages = ["requests"] +symbols = ["requests.get", "requests.post", "requests.request"] + +[[framework]] +id = "httpx" +kind = "library" +languages = ["python"] +packages = ["httpx"] +symbols = ["httpx.get", "httpx.post"] + +[[framework]] +id = "lodash" +kind = "library" +languages = ["javascript", "typescript"] +packages = ["lodash", "lodash.merge", "lodash.set", "lodash.defaultsdeep"] +symbols = ["mergeDeep", "defaultsDeep", "setWith"] + +# The Node HTTP client packages, one group: each is a thin wrapper over `http.request`, and they are known as a set. +[[framework]] +id = "node-http-clients" +kind = "library" +languages = ["javascript", "typescript"] +packages = ["axios", "got", "needle", "request", "superagent", "unirest", "undici"] +symbols = ["axios", "needle", "superagent", "unirest", "undici"] + +[[framework]] +id = "escape-html" +kind = "library" +languages = ["javascript", "typescript"] +packages = ["escape-html"] +symbols = ["escapeHtml"] + +[[framework]] +id = "sanitize-html" +kind = "library" +languages = ["javascript", "typescript"] +packages = ["sanitize-html"] +symbols = ["sanitizeHtml"] + +[[framework]] +id = "dompurify" +kind = "library" +languages = ["javascript", "typescript"] +packages = ["dompurify", "isomorphic-dompurify"] +symbols = ["DOMPurify"] + +[[framework]] +id = "esapi" +kind = "library" +languages = ["java"] +packages = ["org.owasp.esapi:esapi"] +symbols = ["ESAPI", "encodeForHTML", "encodeForURL"] + +[[framework]] +id = "commons-text" +kind = "library" +languages = ["java"] +packages = ["org.apache.commons:commons-text", "org.apache.commons:commons-lang3"] +symbols = ["StringEscapeUtils"] diff --git a/src/openultrasast/ruleset/javascript/rules.toml b/src/openultrasast/ruleset/javascript/rules.toml index c2d0c06..bfba692 100644 --- a/src/openultrasast/ruleset/javascript/rules.toml +++ b/src/openultrasast/ruleset/javascript/rules.toml @@ -104,8 +104,23 @@ status = "enabled" min_evidence_level = "static_corroboration" precision_estimate = 0.0 version = "1" +library = "node-http-clients" pattern = '''\b(axios|fetch|got)\s*[.(][^\n]*\b(?:req|request)\.''' +# Active only when the rule above is dropped (priors off); status and precision are the tagged rule's measurement. +[[rule]] +rule_id = "js-ssrf-language" +title = "JavaScript outbound request with external input needs review (language-level)" +languages = ["javascript", "typescript"] +cwe = "CWE-918" +tags = ["network_entry"] +status = "enabled" +min_evidence_level = "static_corroboration" +precision_estimate = 0.0 +version = "1" +fallback_for = "js-ssrf" +pattern = '''\b(fetch)\s*[.(][^\n]*\b(?:req|request)\.''' + [[rule]] rule_id = "js-weak-hash" title = "JavaScript weak hash needs review" diff --git a/src/openultrasast/ruleset/php/rules.toml b/src/openultrasast/ruleset/php/rules.toml index 2a57747..b46a557 100644 --- a/src/openultrasast/ruleset/php/rules.toml +++ b/src/openultrasast/ruleset/php/rules.toml @@ -26,6 +26,7 @@ status = "enabled" min_evidence_level = "static_corroboration" precision_estimate = 0.032 version = "1" +framework = "wordpress" pattern = '''\$wpdb->(get_var|get_results|get_row|get_col|query|replace)\s*\(\s*(?!\$wpdb->prepare)(?:"[^"\n]*\{?\$(?!wpdb->)[A-Za-z_]|[^\n;]*?["']\s*\.\s*\$(?!wpdb->)[A-Za-z_]|[^\n;]*?\$_(?:GET|POST|REQUEST|COOKIE)\b)''' [[rule]] @@ -54,8 +55,23 @@ status = "enabled" min_evidence_level = "static_corroboration" precision_estimate = 0.034 version = "1" +framework = "wordpress" pattern = '''(?])(?=(VALUES|IN|ORDER|GROUP|LIMIT|OFFSET|WHERE|AND|OR|SET|HAVING)\b)(?:(?:VALUES|IN)\s*\(|(?:ORDER|GROUP)\s+BY\b[^"'\n]*|(?:LIMIT|OFFSET)\s+(?:\d+\s*,\s*)?|(?:WHERE|AND|OR|SET|HAVING)\s+[\w.`]+\s*(?:=|<>|!=|<=|>=|<|>|LIKE)\s*)["']\s*\.\s*\$(?!wpdb->)(?!\w*(?:format|placeholder))[A-Za-z_]''' +# Active only when the rule above is dropped (priors off); status and precision are the tagged rule's measurement. +[[rule]] +rule_id = "php-sql-unquoted-concat-language" +title = "PHP SQL literal concatenates a variable into an unquoted position (language-level)" +languages = ["php"] +cwe = "CWE-89" +tags = ["injection"] +status = "enabled" +min_evidence_level = "static_corroboration" +precision_estimate = 0.034 +version = "1" +fallback_for = "php-sql-unquoted-concat" +pattern = '''(?])(?=(VALUES|IN|ORDER|GROUP|LIMIT|OFFSET|WHERE|AND|OR|SET|HAVING)\b)(?:(?:VALUES|IN)\s*\(|(?:ORDER|GROUP)\s+BY\b[^"'\n]*|(?:LIMIT|OFFSET)\s+(?:\d+\s*,\s*)?|(?:WHERE|AND|OR|SET|HAVING)\s+[\w.`]+\s*(?:=|<>|!=|<=|>=|<|>|LIKE)\s*)["']\s*\.\s*\$(?!\w*(?:format|placeholder))[A-Za-z_]''' + [[rule]] rule_id = "php-sql-unquoted-interpolation" title = "PHP SQL string interpolates a variable into an unquoted position" @@ -66,8 +82,23 @@ status = "enabled" min_evidence_level = "static_corroboration" precision_estimate = 0.129 version = "1" +framework = "wordpress" pattern = '''(?])(?=(VALUES|IN|ORDER|GROUP|LIMIT|OFFSET|WHERE|AND|OR|SET|HAVING)\b)(?:(?:VALUES|IN)\s*\(\s*|(?:ORDER|GROUP)\s+BY\b[^"'\n;]*?|(?:LIMIT|OFFSET)\s+(?:\d+\s*,\s*)?|(?:WHERE|AND|OR|SET|HAVING)\s+[\w.`]+\s*(?:=|<>|!=|<=|>=|<|>|LIKE)\s*)\{?\$(?!wpdb->)(?!\w*(?:format|placeholder))[A-Za-z_]''' +# Active only when the rule above is dropped (priors off); status and precision are the tagged rule's measurement. +[[rule]] +rule_id = "php-sql-unquoted-interpolation-language" +title = "PHP SQL string interpolates a variable into an unquoted position (language-level)" +languages = ["php"] +cwe = "CWE-89" +tags = ["injection"] +status = "enabled" +min_evidence_level = "static_corroboration" +precision_estimate = 0.129 +version = "1" +fallback_for = "php-sql-unquoted-interpolation" +pattern = '''(?])(?=(VALUES|IN|ORDER|GROUP|LIMIT|OFFSET|WHERE|AND|OR|SET|HAVING)\b)(?:(?:VALUES|IN)\s*\(\s*|(?:ORDER|GROUP)\s+BY\b[^"'\n;]*?|(?:LIMIT|OFFSET)\s+(?:\d+\s*,\s*)?|(?:WHERE|AND|OR|SET|HAVING)\s+[\w.`]+\s*(?:=|<>|!=|<=|>=|<|>|LIKE)\s*)\{?\$(?!\w*(?:format|placeholder))[A-Za-z_]''' + # sprintf-built SQL with a quoted %s and no escaping or coercion anywhere in the statement (phpMyFAQ # CVE-2026-46364). Statement-scoped, so it spans lines up to the closing `);`. [[rule]] @@ -80,8 +111,23 @@ status = "enabled" min_evidence_level = "static_corroboration" precision_estimate = 0.109 version = "1" +framework = "wordpress" pattern = '''(sprintf)\(\s*["']\s*(?:SELECT|INSERT|UPDATE|DELETE|REPLACE)\b(?=(?:(?!\);)[\s\S]){0,2000}?'%s')(?=(?:(?![Ee]scape|quote\(|intval\(|absint\(|\(int\)|esc_sql\()[\s\S]){0,2000}?\);)''' +# Active only when the rule above is dropped (priors off); status and precision are the tagged rule's measurement. +[[rule]] +rule_id = "php-sql-sprintf-unescaped-language" +title = "PHP sprintf-built SQL with an unescaped %s value (language-level)" +languages = ["php"] +cwe = "CWE-89" +tags = ["injection"] +status = "enabled" +min_evidence_level = "static_corroboration" +precision_estimate = 0.109 +version = "1" +fallback_for = "php-sql-sprintf-unescaped" +pattern = '''(sprintf)\(\s*["']\s*(?:SELECT|INSERT|UPDATE|DELETE|REPLACE)\b(?=(?:(?!\);)[\s\S]){0,2000}?'%s')(?=(?:(?![Ee]scape|quote\(|intval\(|\(int\))[\s\S]){0,2000}?\);)''' + # --- command / code (command, code_eval) ------------------------------------------------------------------- [[rule]] @@ -120,8 +166,22 @@ status = "shadow" min_evidence_level = "static_corroboration" precision_estimate = 0.25 version = "1" +pattern = '''(?:$\\])(?:$\\])(?:$\\])(?)''' +# Active only when the rule above is dropped (priors off); status and precision are the tagged rule's measurement. +[[rule]] +rule_id = "php-xss-echo-input-language" +title = "PHP echo of request input without HTML escaping (language-level)" +languages = ["php"] +cwe = "CWE-79" +tags = ["injection"] +status = "shadow" +min_evidence_level = "static_corroboration" +precision_estimate = 0.333 +version = "1" +fallback_for = "php-xss-echo-input" +pattern = '''(?m)^(?![^\n]*(?:htmlspecialchars|htmlentities|intval|json_encode|urlencode)\s*\()[^\n]*?(\becho\b|\bprint\b|<\?=)(?:[^\n;]*?[.,])?\s*\$_(?:GET|POST|REQUEST|COOKIE|SERVER)\[[^\]\n]*\]\s*(?:[.,;]|\?>)''' + # --- path / inclusion (filesystem, file_inclusion) --------------------------------------------------------- [[rule]] diff --git a/src/openultrasast/ruleset/python/rules.toml b/src/openultrasast/ruleset/python/rules.toml index b38fb41..ccde301 100644 --- a/src/openultrasast/ruleset/python/rules.toml +++ b/src/openultrasast/ruleset/python/rules.toml @@ -80,6 +80,7 @@ status = "enabled" min_evidence_level = "static_corroboration" precision_estimate = 0.0 version = "1" +framework = "flask" pattern = '''\b(render_template_string)\s*\(''' [[rule]] @@ -92,8 +93,23 @@ status = "enabled" min_evidence_level = "static_corroboration" precision_estimate = 0.0 version = "1" +library = "pyyaml" pattern = '''\b(pickle\.loads|pickle\.load|cPickle\.loads|yaml\.load|marshal\.loads)\s*\(''' +# Active only when the rule above is dropped (priors off); status and precision are the tagged rule's measurement. +[[rule]] +rule_id = "python-unsafe-deserialization-language" +title = "Unsafe Python deserialization needs review (language-level)" +languages = ["python"] +cwe = "CWE-502" +tags = ["deserialization"] +status = "enabled" +min_evidence_level = "static_corroboration" +precision_estimate = 0.0 +version = "1" +fallback_for = "python-unsafe-deserialization" +pattern = '''\b(pickle\.loads|pickle\.load|cPickle\.loads|marshal\.loads)\s*\(''' + [[rule]] rule_id = "python-unsafe-eval" title = "Dynamic Python execution needs review" diff --git a/src/openultrasast/ruleset/semantic/java.toml b/src/openultrasast/ruleset/semantic/java.toml index 8e11a1b..39423b5 100644 --- a/src/openultrasast/ruleset/semantic/java.toml +++ b/src/openultrasast/ruleset/semantic/java.toml @@ -3,7 +3,13 @@ language = "java" [[source]] id = "servlet" -patterns = ["getParameter", "getHeader", "getCookies", "readLine", "getQueryString"] +framework = "jakarta-servlet" +patterns = ["getParameter", "getHeader", "getCookies", "getQueryString"] + +# A request body or a console read line by line: the reader, not the servlet API. +[[source]] +id = "servlet" +patterns = ["readLine"] [[sink]] id = "exec" @@ -45,6 +51,7 @@ rule_ids = [] [[sink]] # CWE-79. Servlet response writers: what reaches these reaches the browser unescaped. id = "response.write" +framework = "jakarta-servlet" cwe = "CWE-79" calls = ["println", "print", "write", "getWriter", "setHeader", "addHeader"] rule_ids = [] @@ -53,13 +60,37 @@ rule_ids = [] # CWE-918 / CWE-601: outbound requests and redirects to a caller-supplied destination. id = "outbound.request" cwe = "CWE-918" -calls = ["openConnection", "openStream", "sendRedirect", "getForObject", "exchange"] +calls = ["openConnection", "openStream"] +rule_ids = [] + +[[sink]] +id = "outbound.request" +framework = "jakarta-servlet" +cwe = "CWE-918" +calls = ["sendRedirect"] +rule_ids = [] + +[[sink]] +id = "outbound.request" +framework = "spring" +cwe = "CWE-918" +calls = ["getForObject", "exchange"] rule_ids = [] [[sanitizer]] # CWE-22 / CWE-79. The documented normalising and encoding APIs; ESAPI is what the OWASP suite itself uses. id = "java.escape" -calls = ["getCanonicalPath", "normalize", "ESAPI.encoder", "encodeForHTML", "encodeForURL", "StringEscapeUtils.escapeHtml4", "URLEncoder.encode"] +calls = ["getCanonicalPath", "normalize", "URLEncoder.encode"] + +[[sanitizer]] +id = "java.escape" +library = "esapi" +calls = ["ESAPI.encoder", "encodeForHTML", "encodeForURL"] + +[[sanitizer]] +id = "java.escape" +library = "commons-text" +calls = ["StringEscapeUtils.escapeHtml4"] [[sanitizer]] id = "parameterized_statement" diff --git a/src/openultrasast/ruleset/semantic/javascript.toml b/src/openultrasast/ruleset/semantic/javascript.toml index 5b6d7e2..e10c1b6 100644 --- a/src/openultrasast/ruleset/semantic/javascript.toml +++ b/src/openultrasast/ruleset/semantic/javascript.toml @@ -3,6 +3,7 @@ language = "javascript" [[source]] id = "req" +framework = "express" patterns = ["req.query", "req.body", "req.params", "req.cookies", "req.headers", "request.query"] [[sink]] @@ -56,14 +57,35 @@ patterns = ["req.url", "request.url", "req.rawHeaders", "url.parse", "parsedUrl" # dynamic key. These are the documented merge helpers plus the assignment forms. id = "prototype.merge" cwe = "CWE-1321" -calls = ["merge", "mergeDeep", "extend", "defaultsDeep", "Object.assign", "set", "setWith"] +calls = ["Object.assign"] +rule_ids = [] + +[[sink]] +id = "prototype.merge" +library = "lodash" +cwe = "CWE-1321" +calls = ["merge", "mergeDeep", "extend", "defaultsDeep", "set", "setWith"] rule_ids = [] [[sink]] # CWE-79. The documented "trusted HTML" sinks in the DOM and the major frameworks. id = "html.unescaped" cwe = "CWE-79" -calls = ["innerHTML", "outerHTML", "insertAdjacentHTML", "document.write", "dangerouslySetInnerHTML", "v-html"] +calls = ["innerHTML", "outerHTML", "insertAdjacentHTML", "document.write"] +rule_ids = [] + +[[sink]] +id = "html.unescaped" +framework = "react" +cwe = "CWE-79" +calls = ["dangerouslySetInnerHTML"] +rule_ids = [] + +[[sink]] +id = "html.unescaped" +framework = "vue" +cwe = "CWE-79" +calls = ["v-html"] rule_ids = [] [[sink]] @@ -79,14 +101,28 @@ rule_ids = [] # limit on this family rather than guessed at here. id = "html.response" cwe = "CWE-79" -calls = ["res.write", "res.end", "res.send", "response.write", "response.end", "response.send"] +calls = ["res.write", "res.end", "response.write", "response.end"] +rule_ids = [] + +[[sink]] +id = "html.response" +framework = "express" +cwe = "CWE-79" +calls = ["res.send", "response.send"] rule_ids = [] [[sink]] # CWE-22. Node filesystem entry points that accept a caller-supplied path. id = "path.open" cwe = "CWE-22" -calls = ["readFile", "readFileSync", "createReadStream", "sendFile", "unlink", "unlinkSync"] +calls = ["readFile", "readFileSync", "createReadStream", "unlink", "unlinkSync"] +rule_ids = [] + +[[sink]] +id = "path.open" +framework = "express" +cwe = "CWE-22" +calls = ["sendFile"] rule_ids = [] [[sink]] @@ -109,13 +145,6 @@ rule_ids = [] id = "outbound.request" cwe = "CWE-918" calls = [ - "axios", # axios, and axios.get/.post/.put/.patch/.delete/.head/.request through it - "got", # got - "needle", # needle - "request", # request / request-promise - "superagent", # superagent - "unirest", # unirest - "undici", # undici, and undici.fetch/.request through it "fetch", # global fetch, node-fetch, cross-fetch "http.get", "http.request", @@ -125,10 +154,34 @@ calls = [ rule_ids = [] prefix_fixes_origin = true +[[sink]] +id = "outbound.request" +library = "node-http-clients" +cwe = "CWE-918" +calls = [ + "axios", # axios, and axios.get/.post/.put/.patch/.delete/.head/.request through it + "got", # got + "needle", # needle + "request", # request / request-promise + "superagent", # superagent + "unirest", # unirest + "undici", # undici, and undici.fetch/.request through it +] +rule_ids = [] +prefix_fixes_origin = true + +[[sink]] +id = "redirect" +framework = "express" +cwe = "CWE-601" +calls = ["redirect"] +rule_ids = [] +prefix_fixes_origin = true + [[sink]] id = "redirect" cwe = "CWE-601" -calls = ["redirect", "location.assign", "location.replace"] +calls = ["location.assign", "location.replace"] rule_ids = [] prefix_fixes_origin = true @@ -146,7 +199,22 @@ calls = ["path.normalize", "path.resolve"] [[sanitizer]] # CWE-79. The standard encoders: the DOM/global URI encoders and the common escaping helpers. id = "html.escape" -calls = ["encodeURIComponent", "encodeURI", "escapeHtml", "escape", "sanitizeHtml", "DOMPurify.sanitize"] +calls = ["encodeURIComponent", "encodeURI", "escape"] + +[[sanitizer]] +id = "html.escape" +library = "escape-html" +calls = ["escapeHtml"] + +[[sanitizer]] +id = "html.escape" +library = "sanitize-html" +calls = ["sanitizeHtml"] + +[[sanitizer]] +id = "html.escape" +library = "dompurify" +calls = ["DOMPurify.sanitize"] [[layout]] diff --git a/src/openultrasast/ruleset/semantic/php.toml b/src/openultrasast/ruleset/semantic/php.toml index f9988e7..de6de92 100644 --- a/src/openultrasast/ruleset/semantic/php.toml +++ b/src/openultrasast/ruleset/semantic/php.toml @@ -46,6 +46,7 @@ patterns = ["php://input"] # them itself, so they are not injection sinks. [[sink]] id = "wpdb" +framework = "wordpress" cwe = "CWE-89" calls = [ "$wpdb->get_var", @@ -124,6 +125,12 @@ rule_ids = [] # --- deserialization --------------------------------------------------------------------------------- +[[sink]] +id = "unserialize" +cwe = "CWE-502" +calls = ["unserialize"] +rule_ids = [] + [[sink]] # `maybe_unserialize` is WordPress's own wrapper and the form every plugin in this project's corpus actually # writes: across four plugins and 1,164 PHP files there is exactly one deserialization site, and it is spelled @@ -134,8 +141,9 @@ rule_ids = [] # It is a real entry point rather than a safe wrapper: `maybe_unserialize` calls `unserialize` whenever the # string looks serialized, and only returns the value untouched when it does not. id = "unserialize" +framework = "wordpress" cwe = "CWE-502" -calls = ["unserialize", "maybe_unserialize"] +calls = ["maybe_unserialize"] rule_ids = [] # --- untrusted destination --------------------------------------------------------------------------- @@ -167,7 +175,14 @@ calls = ["htmlspecialchars", "htmlentities", "html_entity_decode_safe"] # a cleanser. [[sanitizer]] id = "sql_escape" -calls = ["mysqli_real_escape_string", "mysql_real_escape_string", "pg_escape_string", "esc_sql"] +calls = ["mysqli_real_escape_string", "mysql_real_escape_string", "pg_escape_string"] +quoted_only = true + +# WordPress's escape of the same kind. +[[sanitizer]] +id = "sql_escape" +framework = "wordpress" +calls = ["esc_sql"] quoted_only = true [[sanitizer]] @@ -193,6 +208,7 @@ guard = true [[sanitizer]] id = "wpdb_prepare" +framework = "wordpress" calls = ["$wpdb->prepare"] [[sanitizer]] @@ -205,7 +221,12 @@ calls = ["basename", "realpath"] [[sanitizer]] id = "coerce" -calls = ["intval", "floatval", "filter_var", "settype", "absint"] +calls = ["intval", "floatval", "filter_var", "settype"] + +[[sanitizer]] +id = "coerce" +framework = "wordpress" +calls = ["absint"] # A prepared statement is a SHAPE, not a cleansing call: `mysqli_prepare` makes the query safe by binding # rather than by transforming the value, which is the same distinction `execute`/`executemany` carry in @@ -224,6 +245,7 @@ parameterized = true # a row here rather than a change to the query. That is the whole reason it is a fact and not a constant. [[dispatch]] id = "wordpress_hooks" +framework = "wordpress" register = ["add_action", "add_filter", "add_shortcode"] apply = ["apply_filters", "apply_filters_ref_array", "do_action", "do_action_ref_array"] # A filter returns its VALUE argument, filtered; its further arguments are context and never the result, and @@ -240,8 +262,8 @@ request_arguments = ["add_shortcode"] # (a bundled library is a separate unit); `tests` paths stay in scope but are not product, so their # regions are ordered after the product's. Measured before this row: 207 of WP Statistics's 357 PHP files # were vendored and two of them were 60% of its graph; 14 of the 67 regions ahead of MW WP Form's CVE -# were test-mail.php and test-mail-parser.php. +# were test-mail.php and test-mail-parser.php. A product filter (vendored and test paths), not detection: untagged. [[layout]] -id = "php_composer_and_wordpress" +id = "php_layout" vendored = ["vendor/", "node_modules/", "third_party/", "third-party/", "lib/vendor/"] tests = ["tests/", "test/", "spec/", "test-*.php", "*Test.php", "*.test.php", "*_test.php", "phpunit/"] diff --git a/src/openultrasast/ruleset/semantic/python.toml b/src/openultrasast/ruleset/semantic/python.toml index 4a07f87..a16ce1d 100644 --- a/src/openultrasast/ruleset/semantic/python.toml +++ b/src/openultrasast/ruleset/semantic/python.toml @@ -3,6 +3,7 @@ language = "python" [[source]] id = "request" +framework = "flask" patterns = [ "request.args", "request.form", @@ -22,6 +23,7 @@ patterns = [ # validated for type, not for trust. [[source]] id = "django-request" +framework = "django" patterns = [ "request.GET", "request.POST", @@ -54,9 +56,16 @@ rule_ids = ["python-unsafe-eval"] [[sink]] id = "pickle.loads" cwe = "CWE-502" -calls = ["pickle.loads", "pickle.load", "yaml.load", "marshal.loads", "cPickle.loads"] +calls = ["pickle.loads", "pickle.load", "marshal.loads", "cPickle.loads"] rule_ids = ["python-unsafe-deserialization"] +[[sink]] +id = "pickle.loads" +library = "pyyaml" +cwe = "CWE-502" +calls = ["yaml.load"] +rule_ids = [] + [[sink]] id = "execute" cwe = "CWE-89" @@ -82,6 +91,7 @@ calls = ["ast.literal_eval"] [[sanitizer]] id = "yaml.safe_load" +library = "pyyaml" calls = ["yaml.safe_load", "safe_load"] [[sanitizer]] @@ -100,8 +110,16 @@ parameterized = true # CWE-89. Django's documented raw-SQL escape hatches (docs: "Performing raw SQL queries") and the # SQLAlchemy/peewee text forms. These were the single largest miss class in the go/no-go. id = "orm.raw" +framework = "django" cwe = "CWE-89" -calls = ["objects.raw", "raw", "RawSQL", "extra", "text", "from_statement"] +calls = ["objects.raw", "raw", "RawSQL", "extra"] +rule_ids = [] + +[[sink]] +id = "orm.raw" +library = "sqlalchemy" +cwe = "CWE-89" +calls = ["text", "from_statement"] rule_ids = [] [[sink]] @@ -109,7 +127,21 @@ rule_ids = [] # send_file/send_from_directory and Django's FileResponse. id = "path.open" cwe = "CWE-22" -calls = ["open", "send_file", "send_from_directory", "FileResponse", "os.remove", "os.unlink", "shutil.copy", "shutil.move", "shutil.rmtree"] +calls = ["open", "os.remove", "os.unlink", "shutil.copy", "shutil.move", "shutil.rmtree"] +rule_ids = [] + +[[sink]] +id = "path.open" +framework = "flask" +cwe = "CWE-22" +calls = ["send_file", "send_from_directory"] +rule_ids = [] + +[[sink]] +id = "path.open" +framework = "django" +cwe = "CWE-22" +calls = ["FileResponse"] rule_ids = [] [[sink]] @@ -117,8 +149,16 @@ rule_ids = [] # Markup, Django's mark_safe and format_html. Rendering a template *file* is not a sink; rendering a # caller-supplied *string* is. id = "html.unescaped" +framework = "flask" cwe = "CWE-79" -calls = ["render_template_string", "Markup", "mark_safe", "format_html", "HttpResponse"] +calls = ["render_template_string", "Markup"] +rule_ids = [] + +[[sink]] +id = "html.unescaped" +framework = "django" +cwe = "CWE-79" +calls = ["mark_safe", "format_html", "HttpResponse"] rule_ids = [] [[sink]] @@ -126,14 +166,40 @@ rule_ids = [] # destination. requests/urllib/httpx are the documented HTTP clients; redirect is Flask's and Django's. id = "outbound.request" cwe = "CWE-918" -calls = ["requests.get", "requests.post", "requests.request", "urlopen", "urlretrieve", "httpx.get", "httpx.post"] +calls = ["urlopen", "urlretrieve"] +rule_ids = [] +prefix_fixes_origin = true + +[[sink]] +id = "outbound.request" +library = "requests" +cwe = "CWE-918" +calls = ["requests.get", "requests.post", "requests.request"] +rule_ids = [] +prefix_fixes_origin = true + +[[sink]] +id = "outbound.request" +library = "httpx" +cwe = "CWE-918" +calls = ["httpx.get", "httpx.post"] +rule_ids = [] +prefix_fixes_origin = true + +# `redirect` is Flask's (Django's shortcut shares the name; the Flask row carries it). +[[sink]] +id = "redirect" +framework = "flask" +cwe = "CWE-601" +calls = ["redirect", "url_for"] rule_ids = [] prefix_fixes_origin = true [[sink]] id = "redirect" +framework = "django" cwe = "CWE-601" -calls = ["redirect", "HttpResponseRedirect", "url_for"] +calls = ["HttpResponseRedirect"] rule_ids = [] # Django's `reverse` builds an in-site path from a route NAME; request data can only fill its path segments. prefix_fixes_origin = true @@ -143,5 +209,6 @@ origin_anchors = ["reverse", "reverse_lazy"] # redirect is `if not url_has_allowed_host_and_scheme(next_url, ...): next_url = reverse(...)`. [[sanitizer]] id = "url_allowlist" +framework = "django" calls = ["url_has_allowed_host_and_scheme", "is_safe_url"] guard = true diff --git a/src/openultrasast/ruleset/store.py b/src/openultrasast/ruleset/store.py index 33ccf9d..156259e 100644 --- a/src/openultrasast/ruleset/store.py +++ b/src/openultrasast/ruleset/store.py @@ -6,6 +6,8 @@ from collections.abc import Iterable from dataclasses import dataclass, replace from pathlib import Path +from .frameworks import Priors, kept, normalize_priors, read_tag + DEFAULT_RULESET_DIR = Path(__file__).parent VALID_STATUS = ("enabled", "shadow", "disabled") @@ -32,14 +34,23 @@ class PatternRule: min_evidence_level: str = "static_corroboration" precision_estimate: float = 0.0 version: str = "1" + # A prior (learned-decision-engine Req 8.2): the rule knows a framework or a library (`ruleset/frameworks.toml`). + framework: str | None = None + library: str | None = None + # The language-level stand-in of a tagged rule: active only when that rule's prior is off. + fallback_for: str | None = None -def load_ruleset(directory: Path = DEFAULT_RULESET_DIR, ledger: Path | None = None) -> tuple[PatternRule, ...]: +def load_ruleset(directory: Path = DEFAULT_RULESET_DIR, ledger: Path | None = None, *, priors: Priors = "all") -> tuple[PatternRule, ...]: """Load every ``*.toml`` rule file under ``directory`` and apply the loop ledger. Rules are returned sorted by ``rule_id`` for determinism. The optional ledger (``rule_policy.json``) overlays loop-owned ``status``/``min_evidence_level``/ ``precision_estimate`` per ``rule_id``. + + ``priors`` keeps or drops the rules tagged ``framework``/``library``: ``"all"`` (the default, today's rules), + ``"off"`` or a set of ids. A rule with ``fallback_for`` stands in for a tagged rule and is active exactly when + that rule is dropped, so the language-level part of a mixed rule survives with its prior off. """ rules: dict[str, PatternRule] = {} for path in sorted(directory.rglob("*.toml")): @@ -49,8 +60,17 @@ def load_ruleset(directory: Path = DEFAULT_RULESET_DIR, ledger: Path | None = No if rule.rule_id in rules: raise RulesetError(f"duplicate rule_id {rule.rule_id!r} in {path}") rules[rule.rule_id] = rule + chosen = normalize_priors(priors) + for rule in rules.values(): + target = rules.get(rule.fallback_for) if rule.fallback_for else None + if rule.fallback_for and (target is None or not (target.framework or target.library)): + raise RulesetError(f"rule {rule.rule_id}: fallback_for {rule.fallback_for!r} names no tagged rule") + if rule.fallback_for and (rule.framework or rule.library): + raise RulesetError(f"rule {rule.rule_id}: a fallback is language-level and carries no tag") + active = {rule_id: rule for rule_id, rule in rules.items() if kept(rule.framework or rule.library, chosen)} + selected = [rule for rule in active.values() if not (rule.fallback_for and rule.fallback_for in active)] overlay = _load_ledger(ledger) - resolved = [_apply_overlay(rule, overlay.get(rule.rule_id)) for rule in rules.values()] + resolved = [_apply_overlay(rule, overlay.get(rule.rule_id)) for rule in selected] return tuple(sorted(resolved, key=lambda rule: rule.rule_id)) @@ -74,6 +94,7 @@ def write_ruleset(path: Path, rules: Iterable[PatternRule]) -> None: f"min_evidence_level = {_s(rule.min_evidence_level)}", f"precision_estimate = {float(rule.precision_estimate)}", f"version = {_s(rule.version)}", + *(f"{name} = {_s(value)}" for name in ("framework", "library", "fallback_for") if (value := getattr(rule, name))), f"pattern = '''{rule.pattern}'''", ] ) @@ -104,6 +125,10 @@ def _rule_from_dict(item: dict[str, object], path: Path) -> PatternRule: raise RulesetError(f"rule {rule_id} has invalid status {status!r}") if not pattern: raise RulesetError(f"rule {rule_id} has an empty pattern") + try: + framework, library = read_tag(item, f"rule {rule_id} in {path}") + except ValueError as exc: + raise RulesetError(str(exc)) from exc return PatternRule( rule_id=rule_id, title=str(item.get("title", rule_id)), @@ -115,6 +140,9 @@ def _rule_from_dict(item: dict[str, object], path: Path) -> PatternRule: min_evidence_level=str(item.get("min_evidence_level", "static_corroboration")), precision_estimate=float(item.get("precision_estimate", 0.0)), # type: ignore[arg-type] version=str(item.get("version", "1")), + framework=framework, + library=library, + fallback_for=str(item["fallback_for"]) if item.get("fallback_for") else None, ) diff --git a/src/openultrasast/semantic/facts.py b/src/openultrasast/semantic/facts.py index f116a82..f7117c4 100644 --- a/src/openultrasast/semantic/facts.py +++ b/src/openultrasast/semantic/facts.py @@ -1,4 +1,9 @@ -"""Source, sink, and sanitizer facts as versioned data next to the ruleset.""" +"""Source, sink, and sanitizer facts as versioned data next to the ruleset. + +An entry that knows a framework or a library carries ``framework = ""`` or ``library = ""`` (an id of +``ruleset/frameworks.toml``): a prior (learned-decision-engine Req 8.2). :func:`load_facts` keeps or drops the tagged +entries by ``priors`` -- ``"all"`` (the default: today's facts), ``"off"`` (language-level only) or a set of ids. +""" from __future__ import annotations @@ -6,6 +11,8 @@ import tomllib from dataclasses import dataclass from pathlib import Path +from ..ruleset.frameworks import Priors, kept, normalize_priors, read_tag, tag_of + DEFAULT_FACTS_DIR = Path(__file__).resolve().parents[1] / "ruleset" / "semantic" _LANGUAGE_ALIASES = { @@ -31,6 +38,8 @@ class SourceFact: id: str patterns: tuple[str, ...] language: str + framework: str | None = None + library: str | None = None @dataclass(frozen=True) @@ -48,6 +57,8 @@ class SinkFact: # not an untrusted destination, and neither is one that only fills an anchor call's arguments. prefix_fixes_origin: bool = False origin_anchors: tuple[str, ...] = () + framework: str | None = None + library: str | None = None @dataclass(frozen=True) @@ -64,6 +75,8 @@ class SanitizerFact: # makes a value safe only where it held -- inside the branch it guards, past an exit on its failure, or in # the arm of a ternary it selects. Never a cleansing call on a path. guard: bool = False + framework: str | None = None + library: str | None = None @dataclass(frozen=True) @@ -98,6 +111,8 @@ class DispatchFact: # `WP_User` a profile hook gets -- so its parameters are not attacker input; where request data does # reach an apply call's arguments, the hook half of the taint query follows it from there. request_arguments: tuple[str, ...] = () + framework: str | None = None + library: str | None = None @dataclass(frozen=True) @@ -142,6 +157,25 @@ class SemanticFacts: ) +def _tags(item: dict[str, object], path: Path) -> dict[str, str | None]: + framework, library = read_tag(item, f"{path} {item.get('id')!r}") + return {"framework": framework, "library": library} + + +def with_priors(facts: SemanticFacts, priors: Priors) -> SemanticFacts: + """``facts`` with the tagged entries ``priors`` does not keep removed (language-level entries always stay).""" + if priors == "all": + return facts + return SemanticFacts( + version=facts.version, + sources=tuple(f for f in facts.sources if kept(tag_of(f), priors)), + sinks=tuple(f for f in facts.sinks if kept(tag_of(f), priors)), + sanitizers=tuple(f for f in facts.sanitizers if kept(tag_of(f), priors)), + dispatches=tuple(f for f in facts.dispatches if kept(tag_of(f), priors)), + layouts=facts.layouts, + ) + + def _load_facts_uncached(directory: Path | None = None) -> SemanticFacts: root = directory if directory is not None else DEFAULT_FACTS_DIR if not root.is_dir(): @@ -193,6 +227,7 @@ def _dispatches(value: object, language: str, path: Path) -> list[DispatchFact]: returns=_returning(item, path), value_arg=_value_arg(item, path), request_arguments=_request_arguments(item, path), + **_tags(item, path), ) ) return facts @@ -272,7 +307,7 @@ def _sources(value: object, language: str, path: Path) -> list[SourceFact]: patterns = _strings(item.get("patterns"), "patterns", path) if not patterns: raise FactLoadError(f"{path} source {ident} has no patterns") - rows.append(SourceFact(id=ident, patterns=patterns, language=language)) + rows.append(SourceFact(id=ident, patterns=patterns, language=language, **_tags(item, path))) return rows @@ -310,6 +345,7 @@ def _sinks(value: object, language: str, path: Path) -> list[SinkFact]: format_arg=format_arg if isinstance(format_arg, int) else None, prefix_fixes_origin=_flag(item.get("prefix_fixes_origin"), "prefix_fixes_origin", path, ident), origin_anchors=_strings(item.get("origin_anchors"), "origin_anchors", path), + **_tags(item, path), ) ) return rows @@ -336,6 +372,7 @@ def _sanitizers(value: object, language: str, path: Path) -> list[SanitizerFact] literal_format_arg=literal if isinstance(literal, int) else None, quoted_only=_flag(item.get("quoted_only"), "quoted_only", path, ident), guard=_flag(item.get("guard"), "guard", path, ident), + **_tags(item, path), ) ) return rows @@ -372,11 +409,19 @@ def _stat_of(path: Path) -> tuple[int, int]: return info.st_size, info.st_mtime_ns -def load_facts(directory: Path | None = None) -> SemanticFacts: +def load_facts(directory: Path | None = None, *, priors: Priors = "all") -> SemanticFacts: + """The facts of ``directory`` (default: the bundled ruleset), with the priors ``priors`` keeps: ``"all"`` (the + default, today's facts), ``"off"`` or a set of framework/library ids.""" + chosen = normalize_priors(priors) root = directory if directory is not None else DEFAULT_FACTS_DIR if not root.is_dir(): - return _load_facts_uncached(directory) + return with_priors(_load_facts_uncached(directory), chosen) key = _ruleset_state(root, "*.toml") if key not in _FACTS_MEMO: _FACTS_MEMO[key] = _load_facts_uncached(directory) - return _FACTS_MEMO[key] + if chosen == "all": + return _FACTS_MEMO[key] + scoped = (*key, "priors", chosen if isinstance(chosen, str) else tuple(sorted(chosen))) + if scoped not in _FACTS_MEMO: + _FACTS_MEMO[scoped] = with_priors(_FACTS_MEMO[key], chosen) + return _FACTS_MEMO[scoped] diff --git a/tests/test_layout_facts.py b/tests/test_layout_facts.py index 24005e8..c8fc2d5 100644 --- a/tests/test_layout_facts.py +++ b/tests/test_layout_facts.py @@ -11,7 +11,7 @@ def test_layout_rows_load_and_match() -> None: from openultrasast.model.layout import is_test_path, is_vendored, layout_facts rows = layout_facts("php") - assert [r.id for r in rows] == ["php_composer_and_wordpress"] + assert [r.id for r in rows] == ["php_layout"] assert is_vendored("includes/vendor/whichbrowser/parser/data/profiles.php", rows) assert is_vendored("vendor/autoload.php", rows) assert not is_vendored("classes/class.mail.php", rows) diff --git a/tests/test_php_quick_rules.py b/tests/test_php_quick_rules.py index e5677ef..0cfd257 100644 --- a/tests/test_php_quick_rules.py +++ b/tests/test_php_quick_rules.py @@ -31,7 +31,7 @@ def _benchmark(name: str, *, reported_only: bool): # type: ignore[no-untyped-de def test_php_rules_load_compile_and_are_php_scoped() -> None: rules = _php_rules() - assert len(rules) == 13 + assert len(rules) == 14, "13 measured rules; php-unserialize split into its language and WordPress parts (task 2)" for rule in rules.values(): assert rule.languages == ("php",) and rule.status in {"enabled", "shadow"} re.compile(rule.pattern) @@ -52,6 +52,7 @@ def test_every_php_rule_sink_is_in_the_engines_vocabulary() -> None: def test_status_and_precision_are_the_committed_measurement() -> None: measured = json.loads(MEASUREMENT.read_text())["rules"] for rule_id, rule in _php_rules().items(): + rule_id = "php-unserialize" if rule_id == "php-maybe-unserialize" else rule_id # split from it, keeps its numbers assert rule.status == measured[rule_id]["status"], rule_id assert rule.precision_estimate == measured[rule_id]["precision_lower_bound"], rule_id diff --git a/tests/test_ruleset.py b/tests/test_ruleset.py index 08bc05f..7de381b 100644 --- a/tests/test_ruleset.py +++ b/tests/test_ruleset.py @@ -19,7 +19,7 @@ from openultrasast.ruleset import ( def test_default_ruleset_loads_as_data() -> None: rules = load_ruleset(DEFAULT_RULESET_DIR) php = [rule for rule in rules if rule.languages == ("php",)] - assert len(rules) == 32 + len(php) and len(php) == 13 + assert len(rules) == 32 + len(php) and len(php) == 14 # php-unserialize split in two (decision-engine task 2) assert all(rule.status == "enabled" for rule in rules if rule not in php), "PHP rules carry their measured status" assert all(not hasattr(rule, "severity") for rule in rules) # severity is policy-governed, not rule-local diff --git a/tests/test_semantic_priors.py b/tests/test_semantic_priors.py new file mode 100644 index 0000000..f4c21fd --- /dev/null +++ b/tests/test_semantic_priors.py @@ -0,0 +1,166 @@ +"""Framework knowledge as tagged priors (learned-decision-engine task 2; Req 8.2, design section 2). + +The lint: no untagged entry of ``ruleset/semantic/*.toml`` or of a quick ``rules.toml`` contains a symbol of +``ruleset/frameworks.toml`` (a symbol of a framework or library of the entry's language, matched as a whole token; +a quick rule's pattern is read with its regex escapes removed). ``priors="off"`` removes exactly the tagged entries +and activates the language-level fallbacks of mixed quick rules; the default ``"all"`` is today's ruleset. +""" + +from __future__ import annotations + +import re +import tomllib +from pathlib import Path + +import pytest + +from openultrasast.ruleset import DEFAULT_RULESET_DIR, load_ruleset +from openultrasast.ruleset.frameworks import FrameworksError, framework_ids, load_frameworks, symbol_in +from openultrasast.semantic.facts import DEFAULT_FACTS_DIR, load_facts + +TABLES = ("source", "sink", "sanitizer", "dispatch") +LANGUAGE_FAMILY = {"typescript": "javascript", "cpp": "c"} + + +def _lexicon(language: str) -> dict[str, tuple[str, ...]]: + wanted = LANGUAGE_FAMILY.get(language, language) + return {row.id: row.symbols for row in load_frameworks() if wanted in {LANGUAGE_FAMILY.get(lang, lang) for lang in row.languages}} + + +def _texts(item: dict[str, object]) -> list[str]: + keys = ("patterns", "calls", "register", "apply", "returns", "request_arguments", "origin_anchors") + return [str(text) for key in keys for text in item.get(key, []) or []] # type: ignore[union-attr] + + +def _semantic_entries() -> list[tuple[str, str, dict[str, object]]]: + rows = [] + for path in sorted(DEFAULT_FACTS_DIR.glob("*.toml")): + data = tomllib.loads(path.read_text()) + rows += [ + (str(data.get("language") or path.stem), f"{path.name} {table} {item['id']}", item) + for table in TABLES + for item in data.get(table, []) + ] + return rows + + +def _rules() -> list[dict[str, object]]: + return [rule for path in sorted(DEFAULT_RULESET_DIR.glob("*/rules.toml")) for rule in tomllib.loads(path.read_text()).get("rule", [])] + + +def _unescaped(pattern: str) -> str: + return re.sub(r"\\(.)", r"\1", pattern) + + +def test_no_untagged_entry_names_a_framework_symbol() -> None: + offenders: list[str] = [] + for language, where, item in _semantic_entries(): + if item.get("framework") or item.get("library"): + continue + for framework, symbols in _lexicon(language).items(): + hits = sorted({s for s in symbols for text in _texts(item) if symbol_in(s, text)}) + if hits: + offenders.append(f"{where}: {framework} {hits}") + for rule in _rules(): + if rule.get("framework") or rule.get("library"): + continue + text = _unescaped(str(rule["pattern"])) + for language in rule.get("languages") or []: # type: ignore[union-attr] + for framework, symbols in _lexicon(str(language)).items(): + hits = sorted(s for s in symbols if symbol_in(s, text)) + if hits: + offenders.append(f"rule {rule['rule_id']}: {framework} {hits}") + assert not offenders, "untagged framework knowledge (tag it, or split the entry):\n" + "\n".join(sorted(set(offenders))) + + +def test_the_lint_catches_an_untagged_framework_entry() -> None: + item = {"id": "x", "calls": ["$wpdb->get_var", "mysqli_query"]} + assert any(symbol_in(s, t) for s in _lexicon("php")["wordpress"] for t in _texts(item)) + assert not any(symbol_in(s, "maybe_unserialized_value") for s in ("maybe_unserialize",)), "whole tokens only" + assert symbol_in("wpdb->", _unescaped(r"\$(?!wpdb->)[A-Za-z_]")), "a symbol inside an exclusion still counts" + + +def test_every_tag_names_a_listed_id_and_the_lexicon_is_well_formed() -> None: + ids = framework_ids() + tags = [item.get("framework") or item.get("library") for _, _, item in _semantic_entries()] + tags += [rule.get("framework") or rule.get("library") for rule in _rules()] + assert {t for t in tags if t} <= ids + assert {"wordpress", "flask", "django", "express", "spring", "jakarta-servlet"} <= ids + for row in load_frameworks(): + assert row.symbols and row.languages and row.kind in ("framework", "library"), row.id + + +def _signature(facts: object) -> set[tuple[str, str, str, tuple[str, ...]]]: + rows = set() + for kind in ("sources", "sinks", "sanitizers", "dispatches"): + for fact in getattr(facts, kind): + texts = tuple(getattr(fact, "patterns", ()) or getattr(fact, "calls", ()) or getattr(fact, "register", ())) + rows.add((kind, fact.language, fact.id, texts)) + return rows + + +def test_priors_off_removes_exactly_the_tagged_entries() -> None: + every, off = load_facts(), load_facts(priors="off") + assert load_facts(priors="all") is every, "the default is today's facts, memoised" + tagged = { + row for row in _signature(every) + for fact in getattr(every, row[0]) if (fact.language, fact.id) == row[1:3] + and tuple(getattr(fact, "patterns", ()) or getattr(fact, "calls", ()) or getattr(fact, "register", ())) == row[3] + and (fact.framework or fact.library) + } # fmt: skip + assert tagged, "the semantic facts carry tagged priors" + assert _signature(every) - _signature(off) == tagged and _signature(off) <= _signature(every) + assert off.layouts == every.layouts, "layouts are product filters, not detection" + rules_all = {r.rule_id: r for r in load_ruleset(DEFAULT_RULESET_DIR)} + rules_off = {r.rule_id: r for r in load_ruleset(DEFAULT_RULESET_DIR, priors="off")} + tagged_rules = {rid for rid, r in rules_all.items() if r.framework or r.library} + fallbacks = {rid for rid, r in rules_off.items() if r.fallback_for} + assert tagged_rules and set(rules_all) - set(rules_off) == tagged_rules + assert set(rules_off) - set(rules_all) == fallbacks and {rules_off[f].fallback_for for f in fallbacks} <= tagged_rules + assert not any(r.fallback_for for r in rules_all.values()), "a fallback is inactive while its prior is on" + + +def test_split_entries_keep_their_language_level_calls() -> None: + off = load_facts(priors="off") + + def calls(language: str, kind: str, ident: str) -> set[str]: + scoped = off.for_language(language) + return {c for f in getattr(scoped, kind) if f.id == ident for c in (getattr(f, "calls", ()) or getattr(f, "patterns", ()))} + + assert calls("php", "sinks", "unserialize") == {"unserialize"} + assert calls("php", "sanitizers", "sql_escape") == {"mysqli_real_escape_string", "mysql_real_escape_string", "pg_escape_string"} + assert calls("php", "sanitizers", "coerce") == {"intval", "floatval", "filter_var", "settype"} + assert calls("php", "sinks", "wpdb") == set() and not off.for_language("php").dispatches + assert calls("python", "sinks", "pickle.loads") == {"pickle.loads", "pickle.load", "marshal.loads", "cPickle.loads"} + assert calls("python", "sinks", "path.open") == {"open", "os.remove", "os.unlink", "shutil.copy", "shutil.move", "shutil.rmtree"} + assert calls("python", "sinks", "outbound.request") == {"urlopen", "urlretrieve"} and calls("python", "sources", "request") == set() + assert calls("javascript", "sinks", "outbound.request") == {"fetch", "http.get", "http.request", "https.get", "https.request"} + assert calls("javascript", "sinks", "prototype.merge") == {"Object.assign"} + assert calls("javascript", "sanitizers", "html.escape") == {"encodeURIComponent", "encodeURI", "escape"} + assert calls("java", "sinks", "outbound.request") == {"openConnection", "openStream"} + assert calls("java", "sanitizers", "java.escape") == {"getCanonicalPath", "normalize", "URLEncoder.encode"} + assert calls("java", "sources", "servlet") == {"readLine"} + rules = {r.rule_id: r for r in load_ruleset(DEFAULT_RULESET_DIR, priors="off")} + assert "php-unserialize" in rules and "php-maybe-unserialize" not in rules + assert re.search(rules["php-unserialize"].pattern, "$x = unserialize($_POST['a']);") + assert not re.search(rules["php-unserialize"].pattern, "$x = maybe_unserialize($_POST['a']);") + assert re.search(rules["python-unsafe-deserialization-language"].pattern, "pickle.loads(data)") + assert not re.search(rules["python-unsafe-deserialization-language"].pattern, "yaml.load(data)") + + +def test_a_chosen_prior_keeps_only_its_own_entries() -> None: + wordpress = load_facts(priors=frozenset({"wordpress"})) + assert {c for f in wordpress.for_language("php").sinks if f.id == "wpdb" for c in f.calls} >= {"$wpdb->get_var"} + assert not any(f.framework == "flask" for f in wordpress.sources) + rules = {r.rule_id for r in load_ruleset(DEFAULT_RULESET_DIR, priors=frozenset({"wordpress"}))} + assert "php-wpdb-sql-composition" in rules and "python-ssti" not in rules and "python-unsafe-deserialization-language" in rules + with pytest.raises(FrameworksError, match="unknown priors"): + load_facts(priors=frozenset({"no-such-framework"})) + with pytest.raises(FrameworksError): + load_ruleset(DEFAULT_RULESET_DIR, priors="none") # type: ignore[arg-type] + + +def test_a_tag_outside_frameworks_toml_is_refused(tmp_path: Path) -> None: + (tmp_path / "x.toml").write_text('language = "php"\n[[sink]]\nid = "s"\nframework = "nope"\ncwe = "CWE-89"\ncalls = ["q"]\n') + with pytest.raises(ValueError, match="not a framework id"): + load_facts(tmp_path)