marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-11-14 02:08:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(p1): one source-facts module owns function ranges, callers, declaration tables and language aliases

Browse Source 
src/openultrasast/source.py replaces the copies in semantic/functions.py,
plane/tasks/repo_facts.py, learn/features.py, learn/excerpt.py and
plane/tasks/verify.py. Two engines share one sentinel (<module>), one
DECLARATION/KEYWORDS/RESERVED set and one alias map: the parser engine
(ranges, enclosing, bounds, call_sites, callers_of, facts_for) for the
scorer, the overlay and the hunter's annotations; the line-pattern engine
(declared_functions, enclosing_at, function_span, function_bounds, the brace
matcher) that labels, features, roles, excerpts and plane facts were built
on. learn no longer imports plane.tasks. Frozen records that store the old
sentinel are mapped on read (module_function).

Gate: labels rebuilt before and after are identical, 7,921 rows, after
sentinel normalisation (benchmarks/measurements/2026-10-13-source-module-
label-diff/record.json). 490 targeted tests pass; 3,312 collect. Net: 657
lines deleted, 212 added. benchmarks/unseen keeps <global> for its own
frozen manifests.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-13 12:00:37 +02:00
co-authored by Claude Fable 5.1
parent f63703a604
commit 062a7b780e
36 changed files with 1099 additions and 629 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openultrasast/semantic/facts.py
+6 -11
View File
@@ -12,19 +12,10 @@ from dataclasses import dataclass
from pathlib import Path
from ..ruleset.frameworks import Priors, kept, normalize_priors, read_tag, tag_of
from ..source import canonical_language
DEFAULT_FACTS_DIR = Path(__file__).resolve().parents[1] / "ruleset" / "semantic"
_LANGUAGE_ALIASES = {
"c": "c",
"cpp": "c",
"python": "python",
"javascript": "javascript",
"typescript": "javascript",
"java": "java",
}
class FactLoadError(ValueError):
"""Raised when semantic fact files are missing or invalid."""
@@ -146,7 +137,11 @@ class SemanticFacts:
layouts: tuple[LayoutFact, ...] = ()
def for_language(self, language: str) -> SemanticFacts:
key = _LANGUAGE_ALIASES.get(language, language)
key = canonical_language(language)
if key == "cpp":
key = "c"
elif key == "typescript":
key = "javascript"
return SemanticFacts(
version=self.version,
sources=tuple(item for item in self.sources if item.language == key),
src/openultrasast/semantic/functions.py
+6 -147
View File
@@ -1,151 +1,10 @@
"""Named function ranges shared by the scorer, the overlay, and the catalog generator.
"""Function ranges now belong to source."""
Tree-sitter names an arrow function after its first bare parameter and gives
object or class methods, `exports.x = function`, and IIFEs ``<anon>``. Labels in
the pair catalogs name the declarator (``const h = ...``, ``exports.h = ...``,
``h(req, res) {``). Every consumer must therefore name ranges the same way, or
function-scoped matching silently fails (pair-corpus-honesty Req 1.2, design
§Scorer). This module is the single source of that naming. Stdlib plus ``.ir``
only; never imports ``overlay`` or ``pairs``.
"""
from ..source import FunctionRange, enclosing, ranges
from __future__ import annotations
import re
from collections.abc import Sequence
from .ir import FileIR, parse_file
FunctionRange = tuple[str, int, int]
MODULE_FUNCTION = "<module>"
ANON_FUNCTION = "<anon>"
UNNAMED_FUNCTIONS = frozenset({MODULE_FUNCTION, ANON_FUNCTION})
# Keywords a parser or a regex fallback can mistake for a function name; never valid labels.
RESERVED_FUNCTION_LABELS = frozenset(
{
"function",
"def",
"return",
"if",
"else",
"for",
"while",
"switch",
"catch",
"class",
"async",
"await",
"lambda",
"const",
"let",
"var",
"export",
"import",
"module",
"exports",
"new",
"this",
"self",
"anonymous",
ANON_FUNCTION,
}
)
_IDENT = r"[A-Za-z_$][\w$]*"
_ARROW_TAIL = r"(?:async\s*)?(?:\([^)]*\)|" + _IDENT + r")\s*=>"
_DECL = re.compile(
# def name( / function name( / export async function name(
r"^\s*(?:export\s+)?(?:async\s+)?(?:def|function|sub)\s+(" + _IDENT + r")\s*[\({]|"
# const name = [a.b.c =]* [async] (args) => | ident =>
r"^\s*(?:export\s+)?(?:const|let|var)\s+(" + _IDENT + r")\s*=\s*(?:" + _IDENT + r"(?:\." + _IDENT + r")*\s*=\s*)*" + _ARROW_TAIL + r"|"
# exports.name = / module.exports.name = [async] function | (args) =>
r"^\s*(?:module\.)?exports\.(" + _IDENT + r")\s*=\s*(?:async\s*)?(?:function\b|" + _ARROW_TAIL + r")|"
# object property: name: [async] function | (args) =>
r"^\s*(?:async\s+)?(" + _IDENT + r")\s*:\s*(?:async\s*)?(?:function\b|" + _ARROW_TAIL + r")|"
# C / Java / JS method or shorthand: [type] name(args) { (no call: the arg list carries no callback or literal)
r"^\s*(?:[A-Za-z_][\w:<>\*\s,]*[\s\*])?(" + r"[A-Za-z_][\w:$]*" + r")\s*\(([^;{]*)\)\s*(?:const\s*)?\{?\s*$"
)
_CALLBACK_ARGS = re.compile(r"\bfunction\b|=>|['\"`]|^\s*\d")
# A statement keyword before `name(` means the line calls or controls, it does not declare.
_STATEMENT_PREFIX = re.compile(
r"^\s*(?:return|print|yield|throw|await|new|else|case|delete|typeof|synchronized|with|elif|except|try|if|for|while|switch|catch|do)\b"
)
# Only these grammars name an arrow function after its bare parameter; elsewhere the parser name is trustworthy.
_PARAM_NAMED_ARROWS = frozenset({"javascript", "typescript"})
__all__ = ["FunctionRange", "enclosing", "ranges", "function_at"]
def declarator_name(line: str) -> str | None:
"""Function name declared on ``line``, or None when the line declares nothing (a call is not a declaration)."""
match = _DECL.match(line)
if not match:
return None
groups = match.groups()
name = next((group for group in groups[:5] if group), None)
if name is None:
return None
if groups[4] is not None:
args = groups[5] or ""
if _CALLBACK_ARGS.search(args):
return None # `exec('...', function (err) {` is a call, not a declaration
if _STATEMENT_PREFIX.match(line) or name in {"if", "for", "while", "switch", "catch", "return", "elif", "with", "synchronized"}:
return None
name = name.rsplit("::", 1)[-1] # `bool K::m(` declares `m`; catalog labels are unqualified
return name if name not in RESERVED_FUNCTION_LABELS else None
def named_ranges_from_ir(ir: FileIR, text: str) -> tuple[FunctionRange, ...]:
"""Declarator-named function ranges from a parsed file; anonymous functions keep ``<anon>`` only when nothing names them."""
lines = text.splitlines()
ranges: list[FunctionRange] = []
for fn in ir.functions:
if fn.name == MODULE_FUNCTION:
continue
line = lines[fn.start_line - 1] if 0 < fn.start_line <= len(lines) else ""
declared = declarator_name(line)
name = declared or fn.name
if name in RESERVED_FUNCTION_LABELS and name != ANON_FUNCTION:
name = ANON_FUNCTION
elif declared is None and name != ANON_FUNCTION and ir.language in _PARAM_NAMED_ARROWS and not _declares(line, name):
name = ANON_FUNCTION # tree-sitter named a bare-parameter arrow (`block => {`) after its parameter
ranges.append((name, fn.start_line, fn.end_line))
return tuple(ranges)
def _declares(line: str, name: str) -> bool:
"""True when ``line`` visibly declares ``name`` as a function (keyword form or ``name(``), not as a parameter."""
escaped = re.escape(name)
return bool(re.search(r"\b(?:function|def|sub)\s+" + escaped + r"\b|(?<![\w$.])" + escaped + r"\s*\(", line))
def named_function_ranges(path: str, text: str, language: str) -> tuple[FunctionRange, ...] | None:
"""Ranges for one file. None when the parser rejected the file; regex line scan only when no grammar exists."""
ir = parse_file(path, text, language)
if ir.parse_ok:
return named_ranges_from_ir(ir, text)
if ir.reason == "parse_failed":
return None
lines = text.splitlines()
fallback: list[FunctionRange] = []
for index, line in enumerate(lines, start=1):
name = declarator_name(line)
if name:
fallback.append((name, index, len(lines)))
return tuple(fallback)
def function_at(ranges: Sequence[FunctionRange], line: int | None) -> str | None:
"""Innermost *named* function whose range contains ``line``; None when unknown or only anonymous."""
if line is None:
return None
best: FunctionRange | None = None
for item in ranges:
name, start, end = item
if name in UNNAMED_FUNCTIONS or not (start <= line <= end):
continue
if best is None or (end - start) < (best[2] - best[1]):
best = item
return best[0] if best is not None else None
def spans_named(ranges: Sequence[FunctionRange], function: str) -> tuple[tuple[int, int], ...]:
return tuple((start, end) for name, start, end in ranges if name == function)
def function_at(ranges: tuple[FunctionRange, ...], line: int | None) -> str:
"""kept one release; use source.enclosing"""
return enclosing(ranges, line)
src/openultrasast/semantic/obligations/facts.py
+4 -2
View File
@@ -11,6 +11,7 @@ import tomllib
from dataclasses import dataclass
from pathlib import Path
from ...source import canonical_language
from ..facts import _ruleset_state
OPERATION_KINDS = ("protected_read", "protected_write", "privileged_action", "security_setting")
@@ -18,7 +19,6 @@ DISCHARGER_KINDS = ("path_guard", "identity_constraint", "ownership_check", "non
PROVENANCE_KINDS = ("authenticated_context", "request_input", "constant", "unknown")
SENSITIVITIES = ("low", "medium", "high")
DEFAULT_OBLIGATION_FACTS_DIR = Path(__file__).resolve().parents[2] / "ruleset" / "obligations"
_LANGUAGE_ALIASES = {"c_cpp": "c", "cpp": "c", "python_web": "python", "node": "javascript", "ts": "typescript"}
_OPERATION_FIELDS = frozenset({"id", "kind", "calls", "resource_arg", "requires", "sensitivity", "query_document_arg"})
_DISCHARGER_FIELDS = frozenset(
@@ -67,7 +67,9 @@ class ObligationFacts:
dischargers: tuple[DischargerFact, ...]
def for_language(self, language: str) -> ObligationFacts:
key = _LANGUAGE_ALIASES.get(language, language)
key = canonical_language(language)
if key == "cpp":
key = "c"
return ObligationFacts(
version=self.version,
operations=tuple(item for item in self.operations if item.language == key),
src/openultrasast/semantic/obligations/shapes.py
+2 -2
View File
@@ -15,8 +15,8 @@ import re
from collections.abc import Sequence
from dataclasses import dataclass
from ...source import ranges_from_ir
from ..facts import SemanticFacts
from ..functions import named_ranges_from_ir
from ..ir import CallSite, FileIR, FunctionIR, trailing_name
from .facts import DISCHARGER_KINDS, OPERATION_KINDS, PROVENANCE_KINDS, DischargerFact, ObligationFacts, OperationFact
@@ -382,7 +382,7 @@ def _function_named(ir: FileIR, function: str, text: str) -> FunctionIR | None:
if item.name == function:
return item
if text:
for name, start, _end in named_ranges_from_ir(ir, text):
for name, start, _end, _kind in ranges_from_ir(ir, text):
if name == function:
for item in ir.functions:
if item.start_line == start:
src/openultrasast/semantic/overlay.py
+3 -2
View File
@@ -10,9 +10,10 @@ from pathlib import Path
from ..findings import SEVERITY_LABEL, StaticFinding
from ..policy import load_policy, resolve_severity
from ..preprocess import FileTarget
from ..source import FunctionRange, ranges_from_ir
from ..source import enclosing as function_at
from .engines import joern_available, joern_flow
from .facts import FactLoadError, SemanticFacts, load_facts
from .functions import FunctionRange, function_at, named_ranges_from_ir
from .ir import parse_file
from .taint import Demotion, FileFlow, TaintPath, UnadjudicatedFlow, analyze_ir, merge_joern_paths
@@ -172,7 +173,7 @@ def _flows_for(
if not ir.parse_ok:
flows[target.path] = UnadjudicatedFlow(path=target.path, reason=ir.reason or "parse_failed", engine=ir.engine)
continue
ranges[target.path] = named_ranges_from_ir(ir, text)
ranges[target.path] = ranges_from_ir(ir, text)
analyzed: FileFlow | UnadjudicatedFlow = analyze_ir(ir, facts.for_language(target.language))
if isinstance(analyzed, FileFlow):
extra = tuple(joern_extra) if joern_extra is not None else _joern_paths(target.path, text, target.language)