refactor(p1): one source-facts module owns function ranges, callers, declaration tables and language aliases

src/openultrasast/source.py replaces the copies in semantic/functions.py,
plane/tasks/repo_facts.py, learn/features.py, learn/excerpt.py and
plane/tasks/verify.py. Two engines share one sentinel (<module>), one
DECLARATION/KEYWORDS/RESERVED set and one alias map: the parser engine
(ranges, enclosing, bounds, call_sites, callers_of, facts_for) for the
scorer, the overlay and the hunter's annotations; the line-pattern engine
(declared_functions, enclosing_at, function_span, function_bounds, the brace
matcher) that labels, features, roles, excerpts and plane facts were built
on. learn no longer imports plane.tasks. Frozen records that store the old
sentinel are mapped on read (module_function).

Gate: labels rebuilt before and after are identical, 7,921 rows, after
sentinel normalisation (benchmarks/measurements/2026-10-13-source-module-
label-diff/record.json). 490 targeted tests pass; 3,312 collect. Net: 657
lines deleted, 212 added. benchmarks/unseen keeps <global> for its own
frozen manifests.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-13 12:00:37 +02:00
co-authored by Claude Fable 5.1
parent f63703a604
commit 062a7b780e
36 changed files with 1099 additions and 629 deletions
@@ -106,10 +106,10 @@ def _body(lines: list[str], span: tuple[int, int]) -> str:
def extract(source: str, language: str, function: str) -> tuple[list[str], tuple[int, int]] | None:
from openultrasast.learn.excerpt import LANGUAGE_ALIASES, function_bounds
from openultrasast.source import canonical_language, function_bounds
lines = source.splitlines()
span = function_bounds(lines, LANGUAGE_ALIASES.get(language, language), function)
span = function_bounds(lines, canonical_language(language), function)
return (lines, span) if span else None
@@ -159,14 +159,14 @@ def materialise(pair: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
def verify(pair: dict[str, Any]) -> dict[str, Any]:
"""The label builder's check, from the cached excerpts: declared on both sides, body changed, a real fix."""
from openultrasast.learn.excerpt import LANGUAGE_ALIASES, function_bounds
from openultrasast.source import canonical_language, function_bounds
result: dict[str, Any] = {"name": pair["name"], "ok": False, "bytes_read": 0}
commit, parent = str(pair.get("commit", "")), str(pair.get("parent", ""))
if not (_SHA.fullmatch(commit) and _SHA.fullmatch(parent)) or commit == parent:
result["reason"] = "fix commit must be a full SHA distinct from its parent"
return result
language = LANGUAGE_ALIASES.get(str(pair["language"]), str(pair["language"]))
language = canonical_language(str(pair["language"]))
spans = {}
for side, path in zip(("vuln", "fixed"), excerpt_paths(pair), strict=True):
if not path.is_file():
+3 -2
View File
@@ -15,7 +15,8 @@ import tomllib
from pathlib import Path
try:
from openultrasast.semantic.functions import ANON_FUNCTION, named_function_ranges
from openultrasast.source import ANON_FUNCTION
from openultrasast.source import ranges as source_ranges
except ImportError as exc: # never degrade into "no named function": that path deletes corpora under --prune
raise SystemExit(f"catalog_gen needs the openultrasast package (run it with the project venv): {exc}") from exc
@@ -268,7 +269,7 @@ def _language(recipe: dict[str, object]) -> str:
def _ranges(path: Path, language: str) -> tuple[tuple[str, int, int], ...] | None:
ranges = named_function_ranges(str(path), path.read_text(errors="ignore"), language)
ranges = source_ranges(str(path), path.read_text(errors="ignore"), language)
if ranges is None:
return None # parser rejected the excerpt; a regex label would not scope
return tuple(r for r in ranges if r[0] != ANON_FUNCTION)