mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
refactor(p1): one source-facts module owns function ranges, callers, declaration tables and language aliases
src/openultrasast/source.py replaces the copies in semantic/functions.py, plane/tasks/repo_facts.py, learn/features.py, learn/excerpt.py and plane/tasks/verify.py. Two engines share one sentinel (<module>), one DECLARATION/KEYWORDS/RESERVED set and one alias map: the parser engine (ranges, enclosing, bounds, call_sites, callers_of, facts_for) for the scorer, the overlay and the hunter's annotations; the line-pattern engine (declared_functions, enclosing_at, function_span, function_bounds, the brace matcher) that labels, features, roles, excerpts and plane facts were built on. learn no longer imports plane.tasks. Frozen records that store the old sentinel are mapped on read (module_function). Gate: labels rebuilt before and after are identical, 7,921 rows, after sentinel normalisation (benchmarks/measurements/2026-10-13-source-module- label-diff/record.json). 490 targeted tests pass; 3,312 collect. Net: 657 lines deleted, 212 added. benchmarks/unseen keeps <global> for its own frozen manifests. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
f63703a604
commit
062a7b780e
@@ -106,10 +106,10 @@ def _body(lines: list[str], span: tuple[int, int]) -> str:
|
||||
|
||||
|
||||
def extract(source: str, language: str, function: str) -> tuple[list[str], tuple[int, int]] | None:
|
||||
from openultrasast.learn.excerpt import LANGUAGE_ALIASES, function_bounds
|
||||
from openultrasast.source import canonical_language, function_bounds
|
||||
|
||||
lines = source.splitlines()
|
||||
span = function_bounds(lines, LANGUAGE_ALIASES.get(language, language), function)
|
||||
span = function_bounds(lines, canonical_language(language), function)
|
||||
return (lines, span) if span else None
|
||||
|
||||
|
||||
@@ -159,14 +159,14 @@ def materialise(pair: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
|
||||
|
||||
def verify(pair: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The label builder's check, from the cached excerpts: declared on both sides, body changed, a real fix."""
|
||||
from openultrasast.learn.excerpt import LANGUAGE_ALIASES, function_bounds
|
||||
from openultrasast.source import canonical_language, function_bounds
|
||||
|
||||
result: dict[str, Any] = {"name": pair["name"], "ok": False, "bytes_read": 0}
|
||||
commit, parent = str(pair.get("commit", "")), str(pair.get("parent", ""))
|
||||
if not (_SHA.fullmatch(commit) and _SHA.fullmatch(parent)) or commit == parent:
|
||||
result["reason"] = "fix commit must be a full SHA distinct from its parent"
|
||||
return result
|
||||
language = LANGUAGE_ALIASES.get(str(pair["language"]), str(pair["language"]))
|
||||
language = canonical_language(str(pair["language"]))
|
||||
spans = {}
|
||||
for side, path in zip(("vuln", "fixed"), excerpt_paths(pair), strict=True):
|
||||
if not path.is_file():
|
||||
|
||||
@@ -15,7 +15,8 @@ import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from openultrasast.semantic.functions import ANON_FUNCTION, named_function_ranges
|
||||
from openultrasast.source import ANON_FUNCTION
|
||||
from openultrasast.source import ranges as source_ranges
|
||||
except ImportError as exc: # never degrade into "no named function": that path deletes corpora under --prune
|
||||
raise SystemExit(f"catalog_gen needs the openultrasast package (run it with the project venv): {exc}") from exc
|
||||
|
||||
@@ -268,7 +269,7 @@ def _language(recipe: dict[str, object]) -> str:
|
||||
|
||||
|
||||
def _ranges(path: Path, language: str) -> tuple[tuple[str, int, int], ...] | None:
|
||||
ranges = named_function_ranges(str(path), path.read_text(errors="ignore"), language)
|
||||
ranges = source_ranges(str(path), path.read_text(errors="ignore"), language)
|
||||
if ranges is None:
|
||||
return None # parser rejected the excerpt; a regex label would not scope
|
||||
return tuple(r for r in ranges if r[0] != ANON_FUNCTION)
|
||||
|
||||
Reference in New Issue
Block a user