mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 12:39:28 +00:00
refactor(p1): one source-facts module owns function ranges, callers, declaration tables and language aliases
src/openultrasast/source.py replaces the copies in semantic/functions.py, plane/tasks/repo_facts.py, learn/features.py, learn/excerpt.py and plane/tasks/verify.py. Two engines share one sentinel (<module>), one DECLARATION/KEYWORDS/RESERVED set and one alias map: the parser engine (ranges, enclosing, bounds, call_sites, callers_of, facts_for) for the scorer, the overlay and the hunter's annotations; the line-pattern engine (declared_functions, enclosing_at, function_span, function_bounds, the brace matcher) that labels, features, roles, excerpts and plane facts were built on. learn no longer imports plane.tasks. Frozen records that store the old sentinel are mapped on read (module_function). Gate: labels rebuilt before and after are identical, 7,921 rows, after sentinel normalisation (benchmarks/measurements/2026-10-13-source-module- label-diff/record.json). 490 targeted tests pass; 3,312 collect. Net: 657 lines deleted, 212 added. benchmarks/unseen keeps <global> for its own frozen manifests. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
f63703a604
commit
062a7b780e
@@ -32,16 +32,17 @@ import re
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..plane.tasks.repo_facts import DECLARATION, GLOBAL, _declared_name
|
||||
from .features import function_span
|
||||
from ..source import ( # noqa: F401 (re-exported: labels, examples and harvest import them from here)
|
||||
HASH_COMMENTS,
|
||||
SLASH_COMMENTS,
|
||||
_comment_ranges,
|
||||
canonical_language,
|
||||
function_bounds,
|
||||
resolve_language,
|
||||
)
|
||||
from .labels import _ADVISORY, security_reason
|
||||
|
||||
REDACTED = "<redacted>"
|
||||
LANGUAGE_ALIASES = {"c_cpp": "cpp", "c++": "cpp", "js": "javascript", "ts": "typescript", "py": "python"}
|
||||
HASH_COMMENTS = frozenset({"python", "php", "ruby", "shell", "perl", "yaml", "r"})
|
||||
SLASH_COMMENTS = frozenset(
|
||||
{"c", "cpp", "java", "javascript", "typescript", "php", "go", "groovy", "csharp", "kotlin", "rust", "swift", "scala"}
|
||||
) # fmt: skip
|
||||
MAX_LINE_CHARS = 240
|
||||
_HEX = re.compile(r"\b[0-9a-f]{7,40}\b", re.IGNORECASE)
|
||||
|
||||
@@ -87,55 +88,6 @@ def excerpt_sha(text: str) -> str:
|
||||
# --- comments ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _comment_ranges(line: str, state: str | None, language: str) -> tuple[list[tuple[int, int]], str | None]:
|
||||
"""The comment spans of one line and the block state carried to the next (``block`` inside ``/* */``, a triple
|
||||
quote inside a Python docstring). Strings are skipped so a ``#`` in a literal is not a comment."""
|
||||
hashes, slashes, python = language in HASH_COMMENTS, language in SLASH_COMMENTS, language == "python"
|
||||
ranges: list[tuple[int, int]] = []
|
||||
i, n, quote = 0, len(line), None
|
||||
if state is not None:
|
||||
close = "*/" if state == "block" else state
|
||||
j = line.find(close)
|
||||
if j < 0:
|
||||
return [(0, n)], state
|
||||
ranges.append((0, j + len(close)))
|
||||
i, state = j + len(close), None
|
||||
while i < n:
|
||||
c = line[i]
|
||||
if quote:
|
||||
if c == "\\":
|
||||
i += 2
|
||||
continue
|
||||
if c == quote:
|
||||
quote = None
|
||||
i += 1
|
||||
continue
|
||||
if python and line.startswith(('"""', "'''"), i) and not line[:i].strip():
|
||||
delim = line[i : i + 3]
|
||||
j = line.find(delim, i + 3)
|
||||
if j < 0:
|
||||
ranges.append((i, n))
|
||||
return ranges, delim
|
||||
ranges.append((i, j + 3))
|
||||
i = j + 3
|
||||
continue
|
||||
if c in "\"'`":
|
||||
quote = c
|
||||
elif (slashes and line.startswith("//", i)) or (hashes and c == "#"):
|
||||
ranges.append((i, n))
|
||||
break
|
||||
elif slashes and line.startswith("/*", i):
|
||||
j = line.find("*/", i + 2)
|
||||
if j < 0:
|
||||
ranges.append((i, n))
|
||||
return ranges, "block"
|
||||
ranges.append((i, j + 2))
|
||||
i = j + 2
|
||||
continue
|
||||
i += 1
|
||||
return ranges, state
|
||||
|
||||
|
||||
_OPEN = ("/**", "/*", "//", "#", '"""', "'''", "*")
|
||||
_CLOSE = ("*/", '"""', "'''")
|
||||
|
||||
@@ -153,7 +105,7 @@ def _redact_comment(text: str) -> str:
|
||||
def redact(lines: Sequence[str], language: str, identities: Iterable[str] = ()) -> list[str]:
|
||||
"""``lines`` with security wording in comments and advisory ids anywhere replaced by ``<redacted>``, and every
|
||||
identity string (case-insensitive, at least 4 characters) replaced too."""
|
||||
language = LANGUAGE_ALIASES.get(language, language)
|
||||
language = canonical_language(language)
|
||||
names = sorted({i for i in identities if len(i) >= 4}, key=len, reverse=True)
|
||||
identity = re.compile("|".join(re.escape(n) for n in names), re.IGNORECASE) if names else None
|
||||
out: list[str] = []
|
||||
@@ -188,135 +140,6 @@ def identities_of(repo: str = "", path: str = "", commits: Iterable[str] = ()) -
|
||||
# --- spans and excerpts -----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
_UNKNOWN_LANGUAGES = frozenset({"", "other", "unknown"})
|
||||
_PERL_LINE = re.compile(r"^\s*(?:sub\s+\w+\s*[{(;]|use\s+strict\s*;|my\s+[$@%]\w+)")
|
||||
_RUBY_DEF = re.compile(r"^\s*def\s+(?:self\.)?\w+[?!=]?(?:\s*\(|\s|$)")
|
||||
_RUBY_END = re.compile(r"^\s*end\s*$")
|
||||
_BODY_SCAN_LINES = 40 # how far past a declaration head the body's ``{`` (or a prototype's ``;``) may be
|
||||
_STATEMENT_HEAD = re.compile(r"\s*(return|if|while|for|switch|else|case|throw|await|yield)\b")
|
||||
_FUNCTION_EXPR = r"(?:async\s+)?(?:function\b\s*\*?\s*[\w$]*\s*)?(?=\(|[\w$]+\s*=>)" # function (...) / (...) => / a =>
|
||||
|
||||
|
||||
def resolve_language(lines: Sequence[str], language: str) -> str:
|
||||
"""The alias-normalised language; a file given as unknown (``other``, the feature vocabulary's bucket for Go, Ruby,
|
||||
C#, Perl, ...) is sniffed for Perl (``sub name {``, ``use strict;``) and Ruby (``def`` ... ``end``), whose
|
||||
declaration forms a brace matcher never finds and whose ``#`` comments must be redacted."""
|
||||
language = LANGUAGE_ALIASES.get(language, language)
|
||||
if language in _UNKNOWN_LANGUAGES and lines:
|
||||
if any(_PERL_LINE.match(line) for line in lines):
|
||||
return "perl"
|
||||
if any(_RUBY_DEF.match(line) for line in lines) and any(_RUBY_END.match(line) for line in lines):
|
||||
return "ruby"
|
||||
return language
|
||||
|
||||
|
||||
def _blank_comments(lines: Sequence[str], language: str) -> list[str]:
|
||||
style = language if language in SLASH_COMMENTS or language in HASH_COMMENTS else "c"
|
||||
code: list[str] = []
|
||||
state: str | None = None
|
||||
for line in lines:
|
||||
ranges, state = _comment_ranges(line, state, style)
|
||||
for start, end in reversed(ranges):
|
||||
line = line[:start] + " " * (end - start) + line[end:]
|
||||
code.append(line)
|
||||
return code
|
||||
|
||||
|
||||
def _declaration_heads(name: str) -> re.Pattern[str]:
|
||||
"""Where a brace language may declare ``name`` (the body follows): ``name(`` (a definition -- or a call, told
|
||||
apart by :func:`_body_start`); Perl ``sub name {``; an object-literal method ``name: function (`` /
|
||||
``name: (a) =>``; an assignment ``x.y.name = function (`` / ``= (a) =>`` / ``const name = m.name = async a =>``
|
||||
(any chain). ``$`` is an identifier character (``Glance$serveRequest``)."""
|
||||
quoted = re.escape(name)
|
||||
return re.compile(
|
||||
rf"(?<![\w.>$]){quoted}\s*(?:<[^<>(){{}};]*>\s*)?(?=\()"
|
||||
rf"|^\s*sub\s+{quoted}\s*(?=[{{(;])"
|
||||
rf"|(?<![\w$]){quoted}\s*:\s*{_FUNCTION_EXPR}"
|
||||
rf"|(?<![\w$]){quoted}\s*=\s*(?:[\w$.\[\]'\"]+\s*=\s*)*{_FUNCTION_EXPR}"
|
||||
)
|
||||
|
||||
|
||||
def _body_start(code: Sequence[str], line: int, column: int) -> tuple[str, int, int] | None:
|
||||
"""Scan from a declaration head for its body: ``("brace", line, column)`` of the ``{`` at parenthesis depth 0,
|
||||
``("expression", line, column)`` of the ``;`` or ``,`` ending an arrow function's expression body, ``None`` for a
|
||||
prototype (``;`` before any body), a call (a ``)`` closing an outer parenthesis, a ``}``) or no body within
|
||||
:data:`_BODY_SCAN_LINES`. Parentheses are tracked so ``f(a, function () {})`` is not a definition of ``f``."""
|
||||
depth, arrow = 0, False
|
||||
for j in range(line, min(line + _BODY_SCAN_LINES, len(code))):
|
||||
text = code[j]
|
||||
k = column if j == line else 0
|
||||
while k < len(text):
|
||||
c = text[k]
|
||||
if c == "(":
|
||||
depth += 1
|
||||
elif c == ")":
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
return None
|
||||
elif depth == 0:
|
||||
if c == "{":
|
||||
return ("brace", j, k)
|
||||
if arrow and c in ";,":
|
||||
return ("expression", j, k)
|
||||
if c in "};": # a statement ends (a call, a prototype) before any body opened
|
||||
return None
|
||||
if text.startswith("=>", k):
|
||||
arrow = True
|
||||
k += 1
|
||||
k += 1
|
||||
return None
|
||||
|
||||
|
||||
def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
"""A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first
|
||||
declaration head (:func:`_declaration_heads`) whose body :func:`_body_start` finds, to the matching ``}`` (or the
|
||||
end of an expression body). Comments are blanked before anything is matched or counted."""
|
||||
name = re.split(r"::|\.|->", function)[-1]
|
||||
if not re.fullmatch(r"[A-Za-z_$][\w$]*", name):
|
||||
return None
|
||||
heads = _declaration_heads(name)
|
||||
code = _blank_comments(lines, language)
|
||||
for i, line in enumerate(code):
|
||||
if _STATEMENT_HEAD.match(line):
|
||||
continue
|
||||
for found in heads.finditer(line):
|
||||
body = _body_start(code, i, found.end())
|
||||
if body is None:
|
||||
continue
|
||||
kind, j, k = body
|
||||
if kind == "expression":
|
||||
return i, j + 1
|
||||
depth = 0
|
||||
for row in range(j, len(code)):
|
||||
for c in code[row][k:] if row == j else code[row]:
|
||||
if c == "{":
|
||||
depth += 1
|
||||
elif c == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i, row + 1
|
||||
return i, len(code)
|
||||
return None
|
||||
|
||||
|
||||
def function_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
"""0-based ``[start, end)`` of the first declaration of ``function`` (the whole file for ``<global>``), or
|
||||
``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, C#, Perl,
|
||||
...) and brace languages whose pattern misses the name use the brace matcher; an unknown language is sniffed
|
||||
(:func:`resolve_language`)."""
|
||||
language = resolve_language(lines, language)
|
||||
if function == GLOBAL:
|
||||
return (0, len(lines)) if lines else None
|
||||
pattern = DECLARATION.get(language)
|
||||
start = next((i for i, line in enumerate(lines) if _declared_name(pattern, line) == function), None) if pattern else None
|
||||
if start is None:
|
||||
return _brace_bounds(lines, language, function) if language in SLASH_COMMENTS or pattern is None else None
|
||||
span = function_span(lines, language, function)
|
||||
if not span:
|
||||
return None
|
||||
return start, min(start + span, len(lines))
|
||||
|
||||
|
||||
def _window(start: int, end: int, focus: Sequence[int], size: int) -> tuple[int, int]:
|
||||
if end - start <= size:
|
||||
return start, end
|
||||
|
||||
@@ -20,9 +20,11 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..model.taxonomy import load_families
|
||||
from ..plane.tasks.repo_facts import _COMMENT_PREFIXES, DECLARATION, GLOBAL, _declared_name, enclosing, product_files
|
||||
from ..preprocess import build_file_target, detect_language
|
||||
from ..ruleset import PatternRule
|
||||
from ..source import _COMMENT_PREFIXES, DECLARATION, MODULE_FUNCTION, _declared_name, function_span
|
||||
from ..source import enclosing_at as enclosing
|
||||
from ..source import product_paths as product_files
|
||||
from .roles import CWE_OPERATION, SOURCE_KIND, VERSION, Function, RoleSet, body_roles, functions_of, origin_of
|
||||
from .schema import (
|
||||
BY_NAME,
|
||||
@@ -218,35 +220,15 @@ def engine_part(
|
||||
def facts_part(callers: Sequence[Mapping[str, Any]], function: str, version: str | None = None) -> Part:
|
||||
"""``callers`` are the repo-facts call sites of the candidate's function in other files."""
|
||||
files = {str(site.get("path")) for site in callers}
|
||||
return Part("ran", {"facts.callers": len(callers), "facts.caller_files": len(files), "facts.is_global": function == GLOBAL}, version)
|
||||
|
||||
|
||||
def function_span(lines: Sequence[str], language: str, function: str) -> int | None:
|
||||
"""Lines of the first declaration of ``function`` up to the next declaration (Python: up to the next line indented
|
||||
no deeper), ``None`` when the file does not declare it. ``<global>`` counts the whole file."""
|
||||
if function == GLOBAL:
|
||||
return len(lines)
|
||||
pattern = DECLARATION.get(language)
|
||||
if pattern is None:
|
||||
return None
|
||||
start = next((i for i, line in enumerate(lines) if _declared_name(pattern, line) == function), None)
|
||||
if start is None:
|
||||
return None
|
||||
indent = len(lines[start]) - len(lines[start].lstrip(" \t"))
|
||||
for end in range(start + 1, len(lines)):
|
||||
line = lines[end]
|
||||
if language == "python":
|
||||
if line.strip() and not line.lstrip().startswith("#") and len(line) - len(line.lstrip(" \t")) <= indent:
|
||||
return end - start
|
||||
elif _declared_name(pattern, line):
|
||||
return end - start
|
||||
return len(lines) - start
|
||||
return Part("ran", {"facts.callers": len(callers), "facts.caller_files": len(files), "facts.is_global": function == MODULE_FUNCTION},
|
||||
version,
|
||||
)
|
||||
|
||||
|
||||
def source_part(lines: Sequence[str] | None, language: str, function: str) -> Part:
|
||||
"""The function's span; ``none`` without the file or a declaration pattern for its language (C, say: no coverage,
|
||||
not a failure), ``failed`` when a covered file does not declare the function."""
|
||||
if lines is None or (function != GLOBAL and language not in DECLARATION):
|
||||
if lines is None or (function != MODULE_FUNCTION and language not in DECLARATION):
|
||||
return NONE
|
||||
span = function_span(lines, language, function)
|
||||
return Part("ran", {"facts.function_lines": span}) if span is not None else Part("failed")
|
||||
@@ -310,7 +292,7 @@ def model_sinks_part(roles: RoleSet | None, path: str, function: str, version: s
|
||||
|
||||
def function_of(lines: Sequence[str] | None, path: str, language: str, function: str) -> Function | None:
|
||||
"""The declared function ``function`` of a file (its parameters and body), ``None`` when it is not declared."""
|
||||
if not lines or function == GLOBAL:
|
||||
if not lines or function == MODULE_FUNCTION:
|
||||
return None
|
||||
return next((f for f in functions_of(path, lines, language) if f.name == function), None)
|
||||
|
||||
@@ -344,7 +326,7 @@ class SourceIndex:
|
||||
language = detect_language(Path(path))
|
||||
if lines and language in DECLARATION and line and 0 < line <= len(lines):
|
||||
return enclosing(lines, line - 1, language)
|
||||
return fallback or GLOBAL
|
||||
return fallback or MODULE_FUNCTION
|
||||
|
||||
def callers(self) -> dict[str, set[str]]:
|
||||
"""Function name -> the names of the functions that call it anywhere in the product files."""
|
||||
@@ -421,7 +403,7 @@ def _signals(
|
||||
|
||||
def _callers_of(index: SourceIndex, callers: Mapping[str, set[str]], path: str, function: str) -> list[dict[str, Any]]:
|
||||
"""The call sites of ``function`` in other product files, as repo-facts records them (capped at 40)."""
|
||||
if function == GLOBAL or function not in callers:
|
||||
if function == MODULE_FUNCTION or function not in callers:
|
||||
return []
|
||||
sites: list[dict[str, Any]] = []
|
||||
pattern = re.compile(r"(?<![A-Za-z0-9_$])" + re.escape(function) + r"\s*\(")
|
||||
|
||||
@@ -45,9 +45,10 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..model.taxonomy import load_families
|
||||
from ..plane.tasks.repo_facts import _EXCLUDED_DIRS, _NOT_PRODUCT, DECLARATION, GLOBAL, _declared_name, declared_functions, enclosing
|
||||
from ..preprocess import detect_language
|
||||
from ..ruleset.frameworks import load_frameworks
|
||||
from ..source import _EXCLUDED_DIRS, _NOT_PRODUCT, DECLARATION, MODULE_FUNCTION, _declared_name, declared_functions, module_function
|
||||
from ..source import enclosing_at as enclosing
|
||||
|
||||
DEFAULT_SOURCES = Path(__file__).resolve().parent / "sources.toml"
|
||||
DEFAULT_CACHE = Path.home() / ".cache" / "openultrasast"
|
||||
@@ -561,7 +562,9 @@ class LabelBuilder:
|
||||
vulnerable, fixed = str(case["vulnerable"]), str(case["fixed"])
|
||||
old, new = clone.ranges(vulnerable, fixed)
|
||||
if source.options.get("matching") == "declared_sites":
|
||||
positives = {(site.partition("::")[0], site.partition("::")[2]) for site in case.get("sites", []) if "::" in site}
|
||||
positives = {
|
||||
(site.partition("::")[0], module_function(site.partition("::")[2])) for site in case.get("sites", []) if "::" in site
|
||||
}
|
||||
else:
|
||||
positives = changed_functions(clone, vulnerable, old)
|
||||
fixed_changed = changed_functions(clone, fixed, new)
|
||||
@@ -670,12 +673,12 @@ class LabelBuilder:
|
||||
fix (a fix commit distinct from the vulnerable one): exactly one function the fixed side declares that the
|
||||
vulnerable side does not, with the labelled function's parameter list and a changed body (``fixed_side_moved``).
|
||||
A different function of the same snapshot (a guard helper, a sibling handler) is never a negative."""
|
||||
from .excerpt import LANGUAGE_ALIASES, function_bounds
|
||||
from ..source import canonical_language, function_bounds
|
||||
|
||||
fixed_lines, vuln_lines = self._pair_lines(case.fixed_file), self._pair_lines(case.vuln_file)
|
||||
if not fixed_lines:
|
||||
return FixedSide(ABSENT_UNREAD)
|
||||
language = LANGUAGE_ALIASES.get(case.language, case.language)
|
||||
language = canonical_language(case.language)
|
||||
if function_bounds(fixed_lines, language, function):
|
||||
return FixedSide(FIXED_DECLARED)
|
||||
vuln_span = function_bounds(vuln_lines or (), language, function)
|
||||
@@ -721,8 +724,9 @@ class LabelBuilder:
|
||||
if not known.get("in_scope", True) or not known.get("function") or not known.get("file"):
|
||||
self.build.skip("recipes: known entry without an in-scope function")
|
||||
continue
|
||||
candidate = f"{known['file']}::{module_function(str(known['function']))}"
|
||||
self._row(
|
||||
candidate=f"{known['file']}::{known['function']}", family=str(known.get("family", "unknown")), label=1,
|
||||
candidate=candidate, family=str(known.get("family", "unknown")), label=1,
|
||||
source=source.id,
|
||||
source_ref=f"{recipe.get('name')}:{known.get('id', '')}", unit="pin", weight=1.0, split=source.id, group=repo_name(url),
|
||||
repo=url, language=_language(str(known["file"])), evidence="reviewed known vulnerability", pin=commit,
|
||||
@@ -743,11 +747,13 @@ class LabelBuilder:
|
||||
self.build.skip("adjudications: verdict, case or site unreadable")
|
||||
continue
|
||||
population, case = found
|
||||
site_path, line, function = match.group("path"), int(match.group("line")), match.group("function").strip()
|
||||
site_path, line = match.group("path"), int(match.group("line"))
|
||||
function = module_function(match.group("function").strip())
|
||||
if not function:
|
||||
lines = self._clone(population, str(case["id"])).lines(str(case["vulnerable"]), site_path)
|
||||
language = detect_language(Path(site_path))
|
||||
function = enclosing(lines, line - 1, language) if lines and language in DECLARATION and line <= len(lines) else GLOBAL
|
||||
covered = lines and language in DECLARATION and line <= len(lines)
|
||||
function = enclosing(lines, line - 1, language) if covered else MODULE_FUNCTION
|
||||
self._row(
|
||||
candidate=f"{site_path}::{function}", family=str(row.get("family") or case["family"]), label=label, source=source.id,
|
||||
source_ref=f"{Path(path).name}#{key}[{index}]", unit="pin", weight=1.0, split=population.id,
|
||||
|
||||
@@ -31,10 +31,11 @@ from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..plane.tasks.repo_facts import DECLARATION, GLOBAL, KEYWORDS, _declared_name, product_files
|
||||
from ..preprocess import detect_language
|
||||
from ..ruleset.frameworks import Priors, normalize_priors, tag_of
|
||||
from ..semantic.facts import _LANGUAGE_ALIASES, FactLoadError, SanitizerFact, SemanticFacts, SinkFact, SourceFact, load_facts, with_priors
|
||||
from ..semantic.facts import FactLoadError, SanitizerFact, SemanticFacts, SinkFact, SourceFact, load_facts, with_priors
|
||||
from ..source import DECLARATION, KEYWORDS, MODULE_FUNCTION, _declared_name, canonical_language
|
||||
from ..source import product_paths as product_files
|
||||
from .schema import OPERATIONS, ORIGINS, SOURCE_KINDS
|
||||
|
||||
ROLE_KINDS = ("sink", "source", "sanitizer")
|
||||
@@ -128,7 +129,7 @@ class RoleSet:
|
||||
|
||||
def language_key(language: str) -> str:
|
||||
"""The semantic-facts language of a detected language (TypeScript shares JavaScript's entries)."""
|
||||
return _LANGUAGE_ALIASES.get(language, language)
|
||||
return canonical_language(language)
|
||||
|
||||
|
||||
def _facts(language: str, priors: Priors, facts: SemanticFacts | None) -> SemanticFacts:
|
||||
@@ -262,7 +263,7 @@ def functions_of(path: str, lines: Sequence[str], language: str, *, dependency:
|
||||
return []
|
||||
starts: dict[str, int] = {}
|
||||
bodies: dict[str, list[str]] = {}
|
||||
current = GLOBAL
|
||||
current = MODULE_FUNCTION
|
||||
stack: list[tuple[int, str]] = [] # Python: the open (indent, name) declarations, innermost last
|
||||
for index, line in enumerate(lines):
|
||||
declared = _declared_name(pattern, line)
|
||||
@@ -276,11 +277,11 @@ def functions_of(path: str, lines: Sequence[str], language: str, *, dependency:
|
||||
stack.pop()
|
||||
if declared:
|
||||
stack.append((indent, declared))
|
||||
owner = stack[-1][1] if stack else GLOBAL
|
||||
owner = stack[-1][1] if stack else MODULE_FUNCTION
|
||||
else:
|
||||
current = declared or current
|
||||
owner = current
|
||||
if owner != GLOBAL:
|
||||
if owner != MODULE_FUNCTION:
|
||||
bodies.setdefault(owner, []).append(line)
|
||||
out = []
|
||||
for name, start in sorted(starts.items(), key=lambda item: item[1]):
|
||||
@@ -450,7 +451,7 @@ def from_model_roles(payload: Mapping[str, Any]) -> RoleSet:
|
||||
roles.append(
|
||||
Role(
|
||||
detect_language(Path(path)), kind, name or call, operation if operation in vocabulary else "other", "model_role",
|
||||
0, "", path, str(row.get("function") or GLOBAL),
|
||||
0, "", path, str(row.get("function") or MODULE_FUNCTION),
|
||||
)
|
||||
) # fmt: skip
|
||||
return RoleSet(tuple(sorted(set(roles))), tuple(sorted(set(payload.get("unclassified_paths") or ()))), ("model_role",))
|
||||
|
||||
Reference in New Issue
Block a user