mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-19 04:59:28 +00:00
refactor(p1): one source-facts module owns function ranges, callers, declaration tables and language aliases
src/openultrasast/source.py replaces the copies in semantic/functions.py, plane/tasks/repo_facts.py, learn/features.py, learn/excerpt.py and plane/tasks/verify.py. Two engines share one sentinel (<module>), one DECLARATION/KEYWORDS/RESERVED set and one alias map: the parser engine (ranges, enclosing, bounds, call_sites, callers_of, facts_for) for the scorer, the overlay and the hunter's annotations; the line-pattern engine (declared_functions, enclosing_at, function_span, function_bounds, the brace matcher) that labels, features, roles, excerpts and plane facts were built on. learn no longer imports plane.tasks. Frozen records that store the old sentinel are mapped on read (module_function). Gate: labels rebuilt before and after are identical, 7,921 rows, after sentinel normalisation (benchmarks/measurements/2026-10-13-source-module- label-diff/record.json). 490 targeted tests pass; 3,312 collect. Net: 657 lines deleted, 212 added. benchmarks/unseen keeps <global> for its own frozen manifests. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
f63703a604
commit
062a7b780e
+13
-13
@@ -40,8 +40,9 @@ from .rank import rank_targets
|
||||
from .ruleset import PatternRule
|
||||
from .semantic import OverlayRecord, adjudicate
|
||||
from .semantic.facts import FactLoadError, load_facts
|
||||
from .semantic.functions import RESERVED_FUNCTION_LABELS, function_at, named_function_ranges, spans_named
|
||||
from .semantic.obligations.facts import OPERATION_KINDS
|
||||
from .source import MODULE_FUNCTION, RESERVED_FUNCTION_LABELS, FunctionRange, enclosing, spans_named
|
||||
from .source import ranges as source_ranges
|
||||
|
||||
DEFAULT_CATALOG = Path("benchmarks/pairs/catalog.toml")
|
||||
DEFAULT_SAST_CATALOG = Path("benchmarks/pairs/sast/catalog.toml")
|
||||
@@ -57,7 +58,7 @@ DEFAULT_MECHANISMS = Path("benchmarks/pairs/mechanisms.toml")
|
||||
OVERLAY_SLICES = frozenset({"sast", "vfc", "vibe-py", "vfc-js", "agent-vfc", "owasp"})
|
||||
# Harvested function slices: every label must name the function (design §LabelSchema); "<anon>" does not scope.
|
||||
FUNCTION_REQUIRED_SLICES = OVERLAY_SLICES - {"sast"}
|
||||
FunctionRanges = dict[str, tuple[tuple[str, int, int], ...]]
|
||||
FunctionRanges = dict[str, tuple[FunctionRange, ...]]
|
||||
SLICE_NAMES = ("all", "local", "github", "sast", "vfc", "vibe-py", "vfc-js", "agent-vfc", "owasp")
|
||||
PROVENANCES = frozenset({"human", "agent", "mixed", "synthetic"})
|
||||
SPLITS = frozenset({"train", "holdout"})
|
||||
@@ -806,7 +807,7 @@ class _Side:
|
||||
findings: list[StaticFinding]
|
||||
records: list[OverlayRecord]
|
||||
parse_failed: int
|
||||
ranges: dict[str, tuple[tuple[str, int, int], ...]]
|
||||
ranges: dict[str, tuple[FunctionRange, ...]]
|
||||
obligations: list[StaticFinding] = field(default_factory=list) # obligation statics, computed for obligation-labeled rows only
|
||||
|
||||
|
||||
@@ -815,18 +816,17 @@ def _overlay_scan(target: Path, ruleset: tuple[PatternRule, ...] | None = None,
|
||||
findings = [f for f in quick_scan_findings(target, targets, rank_targets(targets), ruleset) if f.status != "shadow"]
|
||||
records = adjudicate(root=target, targets=targets, findings=findings)
|
||||
parse_failed = 0
|
||||
ranges: dict[str, tuple[tuple[str, int, int], ...]] = {}
|
||||
ranges: dict[str, tuple[FunctionRange, ...]] = {}
|
||||
for item in targets:
|
||||
try:
|
||||
text = (target / item.path).read_text(errors="ignore")
|
||||
except OSError:
|
||||
parse_failed += 1
|
||||
continue
|
||||
named = named_function_ranges(item.path, text, item.language)
|
||||
if named is None:
|
||||
try:
|
||||
ranges[item.path] = source_ranges(item.path, text, item.language)
|
||||
except ValueError:
|
||||
parse_failed += 1
|
||||
else:
|
||||
ranges[item.path] = named
|
||||
statics = _obligation_scan(target, targets) if obligations else []
|
||||
return _Side(findings=findings, records=records, parse_failed=parse_failed, ranges=ranges, obligations=statics)
|
||||
|
||||
@@ -890,19 +890,19 @@ def _with_functions(root: Path, findings: list[StaticFinding]) -> list[StaticFin
|
||||
"""Give regex findings an enclosing function from a parse so function-labeled rows can match (Req 1.2)."""
|
||||
if not any(finding.function_name is None for finding in findings):
|
||||
return findings
|
||||
ranges: dict[str, tuple[tuple[str, int, int], ...]] = {}
|
||||
ranges: dict[str, tuple[FunctionRange, ...]] = {}
|
||||
for target in _targets(root):
|
||||
try:
|
||||
named = named_function_ranges(target.path, (root / target.path).read_text(errors="ignore"), target.language)
|
||||
except OSError:
|
||||
named = source_ranges(target.path, (root / target.path).read_text(errors="ignore"), target.language)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if named is not None:
|
||||
ranges[target.path] = named
|
||||
out: list[StaticFinding] = []
|
||||
for finding in findings:
|
||||
if finding.function_name is None:
|
||||
function = function_at(ranges.get(finding.path, ()), finding.line)
|
||||
if function is not None:
|
||||
function = enclosing(ranges.get(finding.path, ()), finding.line)
|
||||
if function != MODULE_FUNCTION:
|
||||
finding = replace(finding, function_name=function)
|
||||
out.append(finding)
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user