From 9d74839128b6e69f936a5d84605f9de0de969fe9 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:12:08 +0200 Subject: [PATCH] fix(learn): excerpt matcher covers more declaration forms The brace matcher now finds a declaration head in more forms -- name(...) with any return type, qualifiers, generics or a parameter list spread over many lines (a paren-aware scan replaces the 8-line lookahead; a prototype's ';' is not a body, a ')' closing an outer parenthesis marks a call), JS object-literal methods and property/chained assignments of function expressions and arrows, $ as an identifier character, Perl 'sub name {', C# expression bodies -- and a file given as 'other'/'unknown' is sniffed for Perl and Ruby so their def/sub forms are matched by the right pattern instead of a {} block inside them. The memory build hands the matcher the label's language where the record's closed vocabulary language ('other' for Go, Ruby, C#) resolves nothing. One test per form. Fixed-side negatives still require the fixed side to declare the function; the 94 vibe-py positives-only groups stay counted. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- src/openultrasast/learn/examples.py | 20 ++- src/openultrasast/learn/excerpt.py | 146 ++++++++++++++---- tests/test_learn_examples.py | 10 ++ tests/test_learn_excerpt.py | 221 ++++++++++++++++++++++++++++ 4 files changed, 363 insertions(+), 34 deletions(-) diff --git a/src/openultrasast/learn/examples.py b/src/openultrasast/learn/examples.py index 4964d85..f9378d0 100644 --- a/src/openultrasast/learn/examples.py +++ b/src/openultrasast/learn/examples.py @@ -122,6 +122,17 @@ def _counter_role(label: Mapping[str, Any]) -> str: return "vulnerable" if label.get("pin_role") == "fixed" else "fixed" +def _excerpt_language(record: Mapping[str, Any], label: Mapping[str, Any], lines: Sequence[str] | None, function: str) -> str: + """The language the excerpt matcher runs under: the record's (the closed feature vocabulary, ``other`` for Go, + Ruby, C#, Perl, ...), and the label's -- the catalog's or the clone's -- where the record's resolves nothing + (a Ruby ``def`` under ``other`` has no brace body). The record keeps its own language field.""" + language = str(record.get("language") or label.get("language") or "other") + declared = str(label.get("language") or "") + if lines and declared and declared != language and function_bounds(lines, language, function) is None: + return declared + return language + + def build_examples( labels: Iterable[Label | Mapping[str, Any]], records: Mapping[tuple[str, str, str, str, str], Mapping[str, Any]], @@ -165,16 +176,17 @@ def build_examples( count.recorded += 1 path, _, function = candidate.partition("::") function = (function_of(label, str(label.get("pin_role") or "vulnerable")) if function_of else None) or function - language = str(record.get("language") or label.get("language") or "other") identities = identities_of(str(label.get("repo") or ""), path, [str(label.get("pin") or "")]) lines = lines_of(label, str(label.get("pin_role") or "vulnerable")) + language = str(record.get("language") or label.get("language") or "other") + matcher = _excerpt_language(record, label, lines, function) diff = None if unit == "delta" and lines: base = lines_of(label, _counter_role(label)) - head_span, base_span = function_bounds(lines, language, function), function_bounds(base or (), language, function) + head_span, base_span = function_bounds(lines, matcher, function), function_bounds(base or (), matcher, function) if base and head_span and base_span: - diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], language, identities=identities) - shown = excerpt(lines, language, function, bounds=bounds, identities=identities, diff=diff) + diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], matcher, identities=identities) + shown = excerpt(lines, matcher, function, bounds=bounds, identities=identities, diff=diff) if shown is None: if not lines: count.unread += 1 diff --git a/src/openultrasast/learn/excerpt.py b/src/openultrasast/learn/excerpt.py index ef9eb94..d788a15 100644 --- a/src/openultrasast/learn/excerpt.py +++ b/src/openultrasast/learn/excerpt.py @@ -14,6 +14,14 @@ otherwise tell the model the label. Both become ````. A delta unit add The text is normalised (trailing blanks dropped, ``\\n`` endings, one final newline) before it is hashed, so the sha names the content an embedding and a response cache key are of. + +Where a function is declared (:func:`function_bounds`): the repo-facts declaration pattern of the language first (its +span runs to the next declaration), else the brace matcher, which knows ``name(...) {`` with any return type, +qualifiers, generics or a parameter list spread over many lines (a prototype's ``;`` is not a body), object-literal +methods ``name: function (`` / ``name: (a) =>``, assignments ``x.y.name = function (`` / ``= (a) =>`` / chained +``const name = m.name = async a =>``, Perl ``sub name {`` and C# expression bodies. A file given as ``other`` or +``unknown`` is sniffed for Perl and Ruby (:func:`resolve_language`), whose ``def``/``sub`` forms a brace matcher +would miss or mistake for a block. """ from __future__ import annotations @@ -180,47 +188,123 @@ def identities_of(repo: str = "", path: str = "", commits: Iterable[str] = ()) - # --- spans and excerpts ----------------------------------------------------------------------------------------------- -def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None: - """A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first - line naming it before ``(`` whose ``{`` comes before any ``;``, to the matching ``}``. Comments are blanked - before braces are counted.""" - name = re.split(r"::|\.|->", function)[-1] - if not re.fullmatch(r"[A-Za-z_]\w*", name): - return None - call = re.compile(r"(?$])" + re.escape(name) + r"\s*\(") +_UNKNOWN_LANGUAGES = frozenset({"", "other", "unknown"}) +_PERL_LINE = re.compile(r"^\s*(?:sub\s+\w+\s*[{(;]|use\s+strict\s*;|my\s+[$@%]\w+)") +_RUBY_DEF = re.compile(r"^\s*def\s+(?:self\.)?\w+[?!=]?(?:\s*\(|\s|$)") +_RUBY_END = re.compile(r"^\s*end\s*$") +_BODY_SCAN_LINES = 40 # how far past a declaration head the body's ``{`` (or a prototype's ``;``) may be +_STATEMENT_HEAD = re.compile(r"\s*(return|if|while|for|switch|else|case|throw|await|yield)\b") +_FUNCTION_EXPR = r"(?:async\s+)?(?:function\b\s*\*?\s*[\w$]*\s*)?(?=\(|[\w$]+\s*=>)" # function (...) / (...) => / a => + + +def resolve_language(lines: Sequence[str], language: str) -> str: + """The alias-normalised language; a file given as unknown (``other``, the feature vocabulary's bucket for Go, Ruby, + C#, Perl, ...) is sniffed for Perl (``sub name {``, ``use strict;``) and Ruby (``def`` ... ``end``), whose + declaration forms a brace matcher never finds and whose ``#`` comments must be redacted.""" + language = LANGUAGE_ALIASES.get(language, language) + if language in _UNKNOWN_LANGUAGES and lines: + if any(_PERL_LINE.match(line) for line in lines): + return "perl" + if any(_RUBY_DEF.match(line) for line in lines) and any(_RUBY_END.match(line) for line in lines): + return "ruby" + return language + + +def _blank_comments(lines: Sequence[str], language: str) -> list[str]: + style = language if language in SLASH_COMMENTS or language in HASH_COMMENTS else "c" code: list[str] = [] state: str | None = None for line in lines: - ranges, state = _comment_ranges(line, state, language if language in SLASH_COMMENTS else "c") + ranges, state = _comment_ranges(line, state, style) for start, end in reversed(ranges): line = line[:start] + " " * (end - start) + line[end:] code.append(line) - for i, line in enumerate(code): - found = call.search(line) - if found is None or re.match(r"\s*(return|if|while|for|switch|else)\b", line): - continue - tail = "\n".join([line[found.end() :], *code[i + 1 : i + 8]]) - brace, semi = tail.find("{"), tail.find(";") - if brace < 0 or (0 <= semi < brace): - continue - depth, opened = 0, False - for j in range(i, len(code)): - for c in line[found.end() :] if j == i else code[j]: + return code + + +def _declaration_heads(name: str) -> re.Pattern[str]: + """Where a brace language may declare ``name`` (the body follows): ``name(`` (a definition -- or a call, told + apart by :func:`_body_start`); Perl ``sub name {``; an object-literal method ``name: function (`` / + ``name: (a) =>``; an assignment ``x.y.name = function (`` / ``= (a) =>`` / ``const name = m.name = async a =>`` + (any chain). ``$`` is an identifier character (``Glance$serveRequest``).""" + quoted = re.escape(name) + return re.compile( + rf"(?$]){quoted}\s*(?:<[^<>(){{}};]*>\s*)?(?=\()" + rf"|^\s*sub\s+{quoted}\s*(?=[{{(;])" + rf"|(? tuple[str, int, int] | None: + """Scan from a declaration head for its body: ``("brace", line, column)`` of the ``{`` at parenthesis depth 0, + ``("expression", line, column)`` of the ``;`` or ``,`` ending an arrow function's expression body, ``None`` for a + prototype (``;`` before any body), a call (a ``)`` closing an outer parenthesis, a ``}``) or no body within + :data:`_BODY_SCAN_LINES`. Parentheses are tracked so ``f(a, function () {})`` is not a definition of ``f``.""" + depth, arrow = 0, False + for j in range(line, min(line + _BODY_SCAN_LINES, len(code))): + text = code[j] + k = column if j == line else 0 + while k < len(text): + c = text[k] + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth < 0: + return None + elif depth == 0: if c == "{": - depth, opened = depth + 1, True - elif c == "}" and opened: - depth -= 1 - if depth == 0: - return i, j + 1 - return i, len(code) + return ("brace", j, k) + if arrow and c in ";,": + return ("expression", j, k) + if c in "};": # a statement ends (a call, a prototype) before any body opened + return None + if text.startswith("=>", k): + arrow = True + k += 1 + k += 1 + return None + + +def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None: + """A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first + declaration head (:func:`_declaration_heads`) whose body :func:`_body_start` finds, to the matching ``}`` (or the + end of an expression body). Comments are blanked before anything is matched or counted.""" + name = re.split(r"::|\.|->", function)[-1] + if not re.fullmatch(r"[A-Za-z_$][\w$]*", name): + return None + heads = _declaration_heads(name) + code = _blank_comments(lines, language) + for i, line in enumerate(code): + if _STATEMENT_HEAD.match(line): + continue + for found in heads.finditer(line): + body = _body_start(code, i, found.end()) + if body is None: + continue + kind, j, k = body + if kind == "expression": + return i, j + 1 + depth = 0 + for row in range(j, len(code)): + for c in code[row][k:] if row == j else code[row]: + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return i, row + 1 + return i, len(code) return None def function_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None: """0-based ``[start, end)`` of the first declaration of ``function`` (the whole file for ````), or - ``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, ...) and - brace languages whose pattern misses the name use the brace matcher.""" - language = LANGUAGE_ALIASES.get(language, language) + ``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, C#, Perl, + ...) and brace languages whose pattern misses the name use the brace matcher; an unknown language is sniffed + (:func:`resolve_language`).""" + language = resolve_language(lines, language) if function == GLOBAL: return (0, len(lines)) if lines else None pattern = DECLARATION.get(language) @@ -267,6 +351,7 @@ def excerpt( :func:`delta_diff`) is appended under a ``diff:`` line.""" if not lines: return None + language = resolve_language(lines, language) span = function_bounds(lines, language, function) if span is None: return None @@ -300,6 +385,7 @@ def delta_diff( ) -> str: """The base->head unified diff of a function (one line of context, no file headers), redacted, at most ``max_lines`` lines.""" + language = resolve_language(head, language) diff = [line for line in difflib.unified_diff(list(base), list(head), lineterm="", n=1)][2:] out: list[str] = [] for line in diff: @@ -314,5 +400,5 @@ def delta_diff( __all__ = [ "CANDIDATE", "DIFF_LINES", "EXAMPLE", "REDACTED", "Bounds", "Excerpt", "delta_diff", "excerpt", "excerpt_sha", - "function_bounds", "identities_of", "normalise", "redact", + "function_bounds", "identities_of", "normalise", "redact", "resolve_language", ] # fmt: skip diff --git a/tests/test_learn_examples.py b/tests/test_learn_examples.py index b9b7319..f3495dd 100644 --- a/tests/test_learn_examples.py +++ b/tests/test_learn_examples.py @@ -158,6 +158,16 @@ def test_a_fixed_side_without_the_function_is_a_label_defect_not_an_unread_side( assert unread.sources["population-v1"].unread == 1 and unread.failing() == ["population-v1"] +def test_the_excerpt_matcher_falls_back_to_the_labels_language_when_the_records_resolves_nothing() -> None: + """A feature record carries the closed vocabulary's language (``other`` for Ruby, Go, C#); a Python ``def`` under + ``other`` has no brace body, so the matcher gets the label's language instead. The record's field is kept.""" + lab = label() + other = {**feature(lab), "language": "other"} + build = build_examples([lab], records_by_key([other]), lambda lab, role: CODE[role], sources=SOURCES) + assert build.summary()["examples"] == 1 and build.rows[0]["language"] == "other" # the row keeps the vocabulary language + assert build.sources["population-v1"].no_excerpt == 0 + + def test_pair_sides_join_the_record_stored_under_their_harvest_unit_pin() -> None: """The harvest stores a pair side's record under its unit pin (the blob sha1 of the side's excerpt), not under :func:`label_pin`; ``pins`` from the harvest ``units.json`` joins them, and without it the record is not found.""" diff --git a/tests/test_learn_excerpt.py b/tests/test_learn_excerpt.py index df9dbdf..be5c1be 100644 --- a/tests/test_learn_excerpt.py +++ b/tests/test_learn_excerpt.py @@ -136,3 +136,224 @@ def test_brace_languages_without_a_declaration_pattern() -> None: assert shown is not None and shown.first == 3 and shown.last == 7 and "other(void)" not in shown.text assert excerpt(c, "cpp", "Parser::parse_header") is not None assert excerpt(["int f(void);", "int g(void) { return f(); }"], "c", "f") is None # a prototype is not a body + + +# --- declaration forms the brace matcher must know (each form once; a call or prototype is never a declaration) ------- + + +def _span(code: list[str], language: str, function: str) -> tuple[int, int] | None: + found = excerpt(code, language, function) + return None if found is None else (found.first, found.last) + + +def test_js_object_literal_methods() -> None: + js = [ + "module.exports = {", + " getInfo: function () {", + " return this.exec('pdfinfo ' + this.path)", + " },", + " ps: function(pid, options, done) {", + " var cmd = 'ps -o pcpu,rss -p ' + pid", + " exec(cmd, done)", + " },", + " performAction: function anonymous(yytext, yyleng) {", + " return eval(yytext)", + " },", + " short: (a) => a + 1,", + " use: function () { return this.getInfo().then(function (info) { return info }) },", + "}", + ] + assert _span(js, "javascript", "getInfo") == (2, 4) + assert _span(js, "javascript", "ps") == (5, 8) + assert _span(js, "javascript", "performAction") == (9, 11) # the key, not the expression's own name + assert _span(js, "javascript", "short") == (12, 12) # an expression body ends at its comma + assert _span(js, "javascript", "then") is None # a call inside a body is not a declaration + + +def test_js_property_assignment_functions_and_arrows() -> None: + js = [ + "module.exports.ConfigureFilePath = (Options, FilePath) => {", + " return path.join(Options.root, FilePath)", + "}", + "const listProcessesOnPort = module.exports.listProcessesOnPort = async port => {", + " return exec(`lsof -i :${port}`)", + "}", + "Glance.prototype.serveRequest = function Glance$serveRequest (req, res) {", + " fs.createReadStream(req.url).pipe(res)", + "}", + "this.getDaqValue = function (tagid, fromts, tots) {", + " return conn.query('SELECT ' + tagid)", + "}", + "const total = sum(prices)", + ] + assert _span(js, "javascript", "ConfigureFilePath") == (1, 3) + assert _span(js, "javascript", "listProcessesOnPort") == (4, 6) + assert _span(js, "javascript", "Glance$serveRequest") == (7, 9) # `$` is an identifier character + assert _span(js, "javascript", "serveRequest") == (7, 9) + assert _span(js, "javascript", "getDaqValue") == (10, 12) + assert _span(js, "javascript", "sum") is None # an assignment of a call result declares nothing + + +def test_ts_class_methods_with_modifiers_and_multiline_parameters() -> None: + ts = [ + "export class Client {", + " private retries = 3", + " private async fetchRetry(", + " url: string,", + " options: any = {},", + " ): Promise {", + " return fetch(url, options)", + " }", + " handle = async (req: Request): Promise => {", + " await this.fetchRetry(req.url)", + " }", + "}", + ] + assert _span(ts, "typescript", "fetchRetry") == (3, 8) # not the `{}` default value inside the parameter list + assert _span(ts, "typescript", "handle") == (9, 11) + + +def test_perl_sub_under_an_unknown_language() -> None: + perl = [ + "#!/usr/bin/perl", + "use strict;", + "sub link_hash_cert;", # a forward declaration has no body + "# CVE-2022-1292: shell injection fixed", + "sub link_hash_cert {", + " my $fname = $_[0];", + " system(\"openssl x509 -in '$fname'\");", + "}", + "sub hash_dir { opendir(my $dh, $_[0]) or die; }", + ] + assert _span(perl, "unknown", "link_hash_cert") == (5, 8) + assert _span(perl, "other", "hash_dir") == (9, 9) + shown = excerpt(perl, "unknown", "link_hash_cert", focus=(4,)) + assert shown is not None and "CVE-2022" not in shown.text # `#` comments are redacted once the language is known + + +def test_c_declaration_with_a_long_parameter_list_and_qualifiers() -> None: + c = [ + "CURLcode Curl_add_custom_headers(struct Curl_easy *data, bool is_connect, void *req);", + "", + "CURLcode Curl_add_custom_headers(struct Curl_easy *data,", + " bool is_connect,", + "#ifndef USE_HYPER", + " struct dynbuf *req", + "#else", + " void *req", + "#endif", + " )", + "{", + " return CURLE_OK;", + "}", + "std::string Parser::header(int index) const noexcept {", + " return headers[index];", + "}", + "Parser::Parser(int n) : size_(n), data_(nullptr) {", + " data_ = alloc(n);", + "}", + "int main(void) { if (check(1)) { return 1; } else if (header(2)) { return 2; } return 0; }", + ] + assert _span(c, "c", "Curl_add_custom_headers") == (3, 13) # the prototype is skipped, the body is 9 lines down + assert _span(c, "cpp", "Parser::header") == (14, 16) + assert _span(c, "cpp", "Parser") == (17, 19) # an initializer list's commas are not statement ends + assert _span(c, "c", "check") is None # `if (check(1)) {` is a call: its `)` closes an outer parenthesis + + +def test_php_methods_with_modifiers_and_return_types() -> None: + php = [ + " None: + go = [ + "package main", + "", + "func (s *Server) handle(w http.ResponseWriter, r *http.Request) {", + "\tio.Copy(w, r.Body)", + "}", + "", + "func main() {}", + ] + assert _span(go, "go", "handle") == (3, 6) # the declaration pattern: to the next `func` + assert _span(go, "other", "handle") == (3, 5) # the feature vocabulary's bucket for Go: brace-matched to the body + rust = ["pub fn parse(input: &mut T) -> Result<(), Error> {", " Ok(())", "}"] + assert _span(rust, "rust", "parse") == (1, 3) + + +def test_ruby_defs_under_the_other_language_are_sniffed() -> None: + ruby = [ + "module Kredis", + " class Type", + " def self.cast_value(value)", + " YAML.load(value)", + " end", + "", + " def fetch_file(path)", + " files.each { |f| return f if f.name == path }", + " nil", + " end", + " end", + "end", + ] + assert _span(ruby, "ruby", "cast_value") == (3, 6) + assert _span(ruby, "other", "cast_value") == (3, 6) # not None: the file is sniffed as Ruby + assert _span(ruby, "other", "fetch_file") == _span(ruby, "ruby", "fetch_file") == (7, 12) # to the end: no later `def` + assert _span(ruby, "c", "fetch_file") == (7, 8) # what a brace matcher takes under a named brace language: the block + + +def test_csharp_attributes_modifiers_and_expression_bodies() -> None: + cs = [ + "public class UploadController : Controller", + "{", + " [HttpPost]", + " public async Task Upload(IFormFile file)", + " {", + " var path = Path.Combine(root, file.FileName);", + " return Ok(path);", + " }", + " protected override string Name(int index) => names[index];", + " public abstract int Size(string key);", + "}", + ] + assert _span(cs, "csharp", "Upload") == (4, 8) + assert _span(cs, "csharp", "Name") == (9, 9) # an expression-bodied member ends at its `;` + assert _span(cs, "csharp", "Size") is None # abstract: no body + + +def test_java_and_statement_heads_are_not_declarations() -> None: + java = [ + "public class Handler {", + " @Override", + " public synchronized List load(String name, Class type)", + " throws IOException, SQLException {", + " return run(name, type);", + " }", + " void other() {", + ' while (load("x", null) != null) { break; }', + " }", + "}", + ] + assert _span(java, "java", "load") == (3, 6) # `throws A, B {`: the comma is not a statement end + assert _span(java, "java", "run") is None + assert _span(java[2:], "java", "load") == _span(java[2:], "c", "load") == (1, 4) # pattern and brace matcher agree + + +def test_resolve_language_sniffs_only_unknown_languages() -> None: + from openultrasast.learn.excerpt import resolve_language + + assert resolve_language(["sub x {", "}"], "unknown") == "perl" + assert resolve_language(["def x", "end"], "other") == "ruby" + assert resolve_language(["def x(a):", " return a"], "other") == "other" # Python has no `end` lines + assert resolve_language(["sub x {", "}"], "python") == "python" # a named language is never second-guessed + assert resolve_language([], "unknown") == "unknown"