refactor(p1): one source-facts module owns function ranges, callers, declaration tables and language aliases

src/openultrasast/source.py replaces the copies in semantic/functions.py,
plane/tasks/repo_facts.py, learn/features.py, learn/excerpt.py and
plane/tasks/verify.py. Two engines share one sentinel (<module>), one
DECLARATION/KEYWORDS/RESERVED set and one alias map: the parser engine
(ranges, enclosing, bounds, call_sites, callers_of, facts_for) for the
scorer, the overlay and the hunter's annotations; the line-pattern engine
(declared_functions, enclosing_at, function_span, function_bounds, the brace
matcher) that labels, features, roles, excerpts and plane facts were built
on. learn no longer imports plane.tasks. Frozen records that store the old
sentinel are mapped on read (module_function).

Gate: labels rebuilt before and after are identical, 7,921 rows, after
sentinel normalisation (benchmarks/measurements/2026-10-13-source-module-
label-diff/record.json). 490 targeted tests pass; 3,312 collect. Net: 657
lines deleted, 212 added. benchmarks/unseen keeps <global> for its own
frozen manifests.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

src/openultrasast/plane/harvest.py

@@ -40,10 +40,10 @@ from typing import Any
 import yaml
 
 from ..preprocess import detect_language
+from ..source import DECLARATION, MODULE_FUNCTION, _declared_name
 from .generate import ATESPACE, CALLS_PER_HUNT, CASE_PATH, INPUTS_PATH, _dump, _json, _task
 from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task
 from .reconciler import _ax_name, _doc
-from .tasks.repo_facts import DECLARATION, GLOBAL, _declared_name
 from .tasks.roles import CHUNK_LINES
 from .tasks.verify import OPERATIONS, units_of
 
@@ -106,9 +106,9 @@ def blob_sha1(text: str) -> str:
 
 
 def declaration_line(lines: Sequence[str] | None, path: str, function: str) -> int:
-    """1-based line of the first declaration of ``function`` in ``lines``; 1 for ``<global>`` or when not declared."""
+    """1-based line of the first declaration of ``function`` in ``lines``; 1 for ``<module>`` or when not declared."""
     pattern = DECLARATION.get(detect_language(Path(path)) or "")
-    if not lines or pattern is None or function == GLOBAL:
+    if not lines or pattern is None or function == MODULE_FUNCTION:
         return 1
     return next((i + 1 for i, line in enumerate(lines) if _declared_name(pattern, line) == function), 1)
 
@@ -116,7 +116,7 @@ def declaration_line(lines: Sequence[str] | None, path: str, function: str) -> i
 def declares(lines: Sequence[str] | None, path: str, function: str) -> bool:
     """Whether a side's file declares ``function``, by the excerpt builder's matcher
     (:func:`..learn.excerpt.function_bounds`): a candidate is asked only where an excerpt of it exists."""
-    from ..learn.excerpt import function_bounds
+    from ..source import function_bounds
 
     return bool(lines) and function_bounds(lines or (), detect_language(Path(path)) or "", function) is not None
 

src/openultrasast/plane/tasks/alerts.py

@@ -9,7 +9,7 @@ what ``ousast scan --mode quick`` runs -- on the case's vulnerable pin and on it
     rule_id, rule_status (enabled | shadow), path, line, function, pin_role (vulnerable | fixed), pin, in_fix_range
 
 ``function`` is the enclosing declaration by the patterns ``repo-facts`` uses (so an alert joins the verify
-candidates' ``path::function``; ``<global>`` outside any function); a language those patterns do not cover keeps the
+candidates' ``path::function``; ``<module>`` outside any function); a language those patterns do not cover keeps the
 scan's own reachability attribution. ``in_fix_range`` says whether the line lies inside the fix: the old-side ranges
 of ``case.json`` on the vulnerable pin, the new-side ``fixed_ranges`` on the fixed pin (null when ``case.json`` has
 none). A fixed-pin alert inside the fix is a false alert on code the fix wrote.
@@ -43,7 +43,8 @@ from ...mapping import analyze_entry_points, attach_reachability_hints
 from ...preprocess import detect_language, preprocess_repository
 from ...rank import rank_targets
 from ...ruleset import DEFAULT_RULESET_DIR, load_ruleset
-from .repo_facts import DECLARATION, GLOBAL, enclosing
+from ...source import DECLARATION, MODULE_FUNCTION
+from ...source import enclosing_at as enclosing
 
 ROLES = ("vulnerable", "fixed")
 DIRS = ("OUSAST_WORKSPACE_DIR", "OUSAST_FIXED_DIR")
@@ -105,12 +106,12 @@ def in_range(ranges: Ranges | None, path: str, line: int | None) -> bool | None:
 
 
 def function_at(root: Path, path: str, line: int | None, fallback: str | None, cache: dict[str, list[str]]) -> str:
-    """The enclosing declaration of ``path:line`` by the ``repo-facts`` patterns, else ``fallback`` (``<global>``)."""
+    """The enclosing declaration of ``path:line`` by the ``repo-facts`` patterns, else ``fallback`` (``<module>``)."""
     lines = cache.setdefault(path, _lines(root / path))
     language = detect_language(root / path)
     if language in DECLARATION and line and 0 < line <= len(lines):
         return enclosing(lines, line - 1, language)
-    return fallback or GLOBAL
+    return fallback or MODULE_FUNCTION
 
 
 def scan(root: Path, role: str, pin: str, ranges: Ranges | None) -> tuple[list[dict[str, Any]], dict[str, Any]]:

src/openultrasast/plane/tasks/features.py

@@ -55,8 +55,8 @@ from ...learn.roles import from_model_roles, infer_for_checkout
 from ...model.taxonomy import load_families
 from ...preprocess import detect_language
 from ...ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset
+from ...source import MODULE_FUNCTION
 from .alerts import covers
-from .repo_facts import GLOBAL
 
 OUTPUT = "features.jsonl"
 PASSES = ("a", "b", "c")
@@ -142,7 +142,7 @@ def case_features(
         rule_id = str(alert.get("rule_id") or "")
         engine = rule_id.startswith(ENGINE_PREFIX)
         family = rule_id.removeprefix(ENGINE_PREFIX) if engine else _family(by_id.get(rule_id))
-        key = (f"{alert.get('path')}::{alert.get('function') or GLOBAL}", family)
+        key = (f"{alert.get('path')}::{alert.get('function') or MODULE_FUNCTION}", family)
         entry = candidates.setdefault(key, {"line": alert.get("line"), "quick": [], "engine": []})
         if engine:
             entry["engine"].append({"site": f"{alert.get('path')}:{alert.get('line')}"})
@@ -179,7 +179,7 @@ def case_features(
                 entries = entry_names(index.root, {c.partition("::")[0] for c, _ in candidates})
             parts["source"] = source_part(index.lines(path), language, function)
             parts["roles"] = roles_part(function_of(index.lines(path), path, language, function), inferred, language)
-            distance = 0 if function == GLOBAL and path in entries[1] else entry_distance(function, entries[0], index.callers())
+            distance = 0 if function == MODULE_FUNCTION and path in entries[1] else entry_distance(function, entries[0], index.callers())
             parts["entry_points"] = Part("ran", {"facts.entry_distance": distance})
         out.append(record(candidate, family, language, "plane", parts))
     return out

src/openultrasast/plane/tasks/repo_facts.py

@@ -13,172 +13,39 @@ declare them.
 
 from __future__ import annotations
 
-import hashlib
 import json
 import os
-import re
 import sys
 import traceback
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Mapping, Sequence
 from pathlib import Path
 from typing import Any
 
 import yaml
 
-from ...model.layout import is_test_path, is_vendored, layout_facts
-from ...preprocess import IGNORED_DIRS, detect_language
+from ...preprocess import detect_language
+from ...source import (  # noqa: F401  (re-exported: generate, remember and the tests import them from here)
+    DECLARATION,
+    KEYWORDS,
+    MODULE_FUNCTION,
+    _declared_name,
+    _digest,
+    declared_functions,
+)
+from ...source import call_sites_in as call_sites
+from ...source import enclosing_at as enclosing
+from ...source import product_paths as product_files
 
 CALLERS_CAP = 40
-MAX_FILE_BYTES = 1_000_000
-LANGUAGES = frozenset({"python", "php", "javascript", "typescript", "java", "go", "ruby"})
-GLOBAL = "<global>"
 
-_EXCLUDED_DIRS = frozenset(
-    IGNORED_DIRS | {"vendor", "node_modules", "bower_components", "third_party", "third-party", "dist", "build", "out", "target"}
-)
-_NOT_PRODUCT = re.compile(
-    r"(^|/)(tests?|__tests__|__mocks__|spec|specs|docs?|examples?|fixtures?|e2e|cypress|storybook|testdata)/"
-    r"|\.(test|spec|stories|mock|e2e)\.[cm]?[jt]sx?$"
-    r"|_test\.(py|go|rb)$|(^|/)test_[^/]*\.(py|rb)$|Test\.java$|Tests\.java$|_spec\.rb$"
-    r"|\.min\.(js|css)$|\.bundle\.js$"
-)
 
-_JS_MODIFIERS = r"(?:(?:export|default|public|private|protected|static|async|readonly|override)\s+)*"
-_JAVA_MODIFIERS = r"(?:(?:public|private|protected|static|final|abstract|synchronized|native|default|strictfp)\s+)*"
-DECLARATION: dict[str, re.Pattern[str]] = {
-    "python": re.compile(r"^\s*(?:async\s+)?def\s+(\w+)\s*\("),
-    "php": re.compile(r"^\s*(?:(?:public|private|protected|static|final|abstract)\s+)*function\s+&?(\w+)\s*\("),
-    "javascript": re.compile(
-        r"^\s*" + _JS_MODIFIERS + r"function\s*\*?\s*(\w+)\s*\("
-        r"|^\s*" + _JS_MODIFIERS + r"(\w+)\s*\([^;]*\)\s*(?::[^{=]+)?\{\s*$"
-        r"|^\s*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?(?:function\b|\([^)]*\)\s*(?::[^=]+)?=>|\w+\s*=>)"
-    ),
-    "java": re.compile(
-        r"^\s*" + _JAVA_MODIFIERS + r"(?:<[^>]*>\s+)?(?!return\b|new\b|throw\b)[\w<>\[\],.?]+\s+(\w+)\s*\([^;]*\)\s*"
-        r"(?:throws\s+[\w.,\s]+)?\{?\s*$"
-    ),
-    "go": re.compile(r"^\s*func\s+(?:\([^)]*\)\s*)?(\w+)\s*\("),
-    "ruby": re.compile(r"^\s*def\s+(?:self\.)?(\w+[?!=]?)"),
-}
 DECLARATION["typescript"] = DECLARATION["javascript"]
-KEYWORDS = frozenset(
-    {
-        "if",
-        "for",
-        "while",
-        "switch",
-        "catch",
-        "return",
-        "function",
-        "else",
-        "do",
-        "try",
-        "with",
-        "elif",
-        "except",
-        "new",
-        "throw",
-        "super",
-        "constructor",
-        "foreach",
-        "match",
-        "case",
-        "unless",
-        "until",
-    }
-)
-_COMMENT_PREFIXES = ("#", "//", "/*", "*", "*/", "--")
 
 
 class TaskFailed(RuntimeError):
     """A crash inside `run` that has already been recorded in `summary.json` (status `failed`, traceback)."""
 
 
-def product_files(workspace: Path) -> list[Path]:
-    """Product source files of the supported languages, sorted; tests, vendored trees, build output, minified
-    bundles and dot-directories are not product (the classification of `model_sinks.py` and the layout facts)."""
-    facts = layout_facts()
-    files: list[Path] = []
-    for path in sorted(workspace.rglob("*")):
-        if path.is_symlink() or not path.is_file():
-            continue
-        relative = path.relative_to(workspace)
-        parts = relative.parts
-        if any(part.startswith(".") or part in _EXCLUDED_DIRS for part in parts[:-1]) or parts[-1].startswith("."):
-            continue
-        posix = relative.as_posix()
-        if _NOT_PRODUCT.search(posix) or is_vendored(posix, facts) or is_test_path(posix, facts):
-            continue
-        if detect_language(path) not in LANGUAGES or path.stat().st_size > MAX_FILE_BYTES:
-            continue
-        files.append(path)
-    return files
-
-
-def declared_functions(lines: Sequence[str], language: str) -> list[str]:
-    """The function names a file declares, in source order, each once."""
-    pattern = DECLARATION.get(language)
-    if pattern is None:
-        return []
-    names: list[str] = []
-    for line in lines:
-        name = _declared_name(pattern, line)
-        if name and name not in names:
-            names.append(name)
-    return names
-
-
-def enclosing(lines: Sequence[str], index: int, language: str) -> str:
-    """The nearest declaration at or above line ``index`` (0-based), `<global>` when there is none."""
-    pattern = DECLARATION.get(language)
-    if pattern is None:
-        return GLOBAL
-    indent = _indent(lines[index]) if language == "python" else None
-    for back in range(index, -1, -1):
-        name = _declared_name(pattern, lines[back])
-        if not name:
-            continue
-        if indent is None or back == index or _indent(lines[back]) < indent:
-            return name
-        indent = min(indent, _indent(lines[back]))  # a sibling or deeper def: keep looking for the one that encloses
-    return GLOBAL
-
-
-def _indent(line: str) -> int:
-    return len(line) - len(line.lstrip(" \t"))
-
-
-def call_sites(lines: Sequence[str], language: str, names: Iterable[str]) -> list[dict[str, Any]]:
-    """Calls to any of ``names`` in one file: `{"function", "line", "enclosing"}`, one per name and line, in
-    line order. A declaration line is not a call of the function it declares; comment lines are skipped."""
-    wanted = sorted(set(names), key=lambda n: (-len(n), n))
-    if not wanted:
-        return []
-    token = re.compile(r"(?<![A-Za-z0-9_$])(" + "|".join(re.escape(n) for n in wanted) + r")\s*\(")
-    pattern = DECLARATION.get(language)
-    found: list[dict[str, Any]] = []
-    for index, line in enumerate(lines):
-        if line.lstrip().startswith(_COMMENT_PREFIXES):
-            continue
-        declared = _declared_name(pattern, line) if pattern is not None else ""
-        seen: set[str] = set()
-        for match in token.finditer(line):
-            name = match.group(1)
-            if name == declared or name in seen:
-                continue
-            seen.add(name)
-            found.append({"function": name, "line": index + 1, "enclosing": enclosing(lines, index, language)})
-    return found
-
-
-def _declared_name(pattern: re.Pattern[str], line: str) -> str:
-    match = pattern.match(line)
-    if not match:
-        return ""
-    name = next((g for g in match.groups() if g), "")
-    return "" if name in KEYWORDS else name
-
-
 def _read_lines(path: Path) -> list[str]:
     return path.read_text(encoding="utf-8", errors="replace").splitlines()
 
@@ -187,10 +54,6 @@ def _candidate_names(candidates: Sequence[Mapping[str, Any]]) -> list[str]:
     return sorted({str(c["function"]) for c in candidates if c.get("function")})
 
 
-def _digest(names: Sequence[str]) -> str:
-    return hashlib.sha256("\n".join(names).encode("utf-8")).hexdigest()[:16]
-
-
 def _load_units(units_path: Path, digest: str) -> dict[str, dict[str, Any]]:
     """Rows already in `units.jsonl` that were computed for the same candidate set (Requirement 2.2)."""
     rows: dict[str, dict[str, Any]] = {}
@@ -240,7 +103,7 @@ def assemble_facts(
     for path, row in sorted(rows.items()):
         for call in row.get("calls", ()):
             by_name.setdefault(str(call["function"]), []).append(
-                {"path": path, "line": int(call["line"]), "enclosing": str(call.get("enclosing", GLOBAL))}
+                {"path": path, "line": int(call["line"]), "enclosing": str(call.get("enclosing", MODULE_FUNCTION))}
             )
     callers: dict[str, list[dict[str, Any]]] = {}
     for candidate in sorted(candidates, key=lambda c: (str(c["path"]), str(c["function"]))):

src/openultrasast/plane/tasks/roles.py

@@ -32,9 +32,9 @@ from typing import Any
 
 from ...learn.roles import MODEL_ROLE_KINDS
 from ...learn.schema import OPERATIONS, SOURCE_KINDS
+from ...source import MODULE_FUNCTION, product_files
 from ...tool_hunter import ChatClient
 from ..budget import AccountError, BudgetExhausted, MeteredClient
-from .repo_facts import GLOBAL, product_files
 from .verify import build_client
 
 CHUNK_LINES = 400
@@ -52,7 +52,7 @@ PROMPT = (
     "- guard: checks a value and rejects it (allow-list, permission or ownership check);\n"
     "- dispatch: registers or invokes a callback through a string-keyed registry (hooks, events, signals).\n"
     "Answer with JSON only: "
-    '{{"roles": [{{"role": "<sink|source|sanitizer|guard|dispatch>", "function": "<enclosing function, or <global>>", '
+    '{{"roles": [{{"role": "<sink|source|sanitizer|guard|dispatch>", "function": "<enclosing function, or <module>>", '
     '"line": <line number>, "call": "<the call or expression>", "operation": "<the kind>", '
     '"evidence": "<the line, quoted>"}}]}}. Return {{"roles": []}} if there is none.\n\nFile: {path}\n\n{code}'
 )
@@ -80,7 +80,7 @@ def parse_roles(content: str | None, path: str) -> list[dict[str, Any]]:
         vocabulary = SOURCE_KINDS if item["role"] == "source" else OPERATIONS
         operation = item.get("operation") if item.get("operation") in vocabulary else "other"
         named = item.get("function")
-        function = named.strip() if isinstance(named, str) and named.strip() else GLOBAL
+        function = named.strip() if isinstance(named, str) and named.strip() else MODULE_FUNCTION
         line = item["line"] if isinstance(item.get("line"), int) else 0
         out.append(
             {
@@ -154,7 +154,7 @@ def run(
     log = output_dir / "units.jsonl"
     metered = budget if budget is not None else MeteredClient(client)
     model = model or "unbound"
-    wanted = sorted(set(files)) if files is not None else [p.relative_to(workspace).as_posix() for p in product_files(workspace)]
+    wanted = sorted(set(files)) if files is not None else product_files(workspace)
     missing = [p for p in wanted if not (workspace / p).is_file()]
     done = {p: row for p, row in _read_units(log).items() if p in wanted}
     status, reason = ("failed", f"files not in the workspace: {missing[:5]}") if missing else ("done", "")