fix(learn): excerpt matcher covers more declaration forms

The brace matcher now finds a declaration head in more forms -- name(...) with
any return type, qualifiers, generics or a parameter list spread over many
lines (a paren-aware scan replaces the 8-line lookahead; a prototype's ';' is
not a body, a ')' closing an outer parenthesis marks a call), JS object-literal
methods and property/chained assignments of function expressions and arrows,
$ as an identifier character, Perl 'sub name {', C# expression bodies -- and a
file given as 'other'/'unknown' is sniffed for Perl and Ruby so their def/sub
forms are matched by the right pattern instead of a {} block inside them. The
memory build hands the matcher the label's language where the record's closed
vocabulary language ('other' for Go, Ruby, C#) resolves nothing. One test per
form. Fixed-side negatives still require the fixed side to declare the
function; the 94 vibe-py positives-only groups stay counted.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

src/openultrasast/learn/examples.py

@@ -122,6 +122,17 @@ def _counter_role(label: Mapping[str, Any]) -> str:
     return "vulnerable" if label.get("pin_role") == "fixed" else "fixed"
 
 
+def _excerpt_language(record: Mapping[str, Any], label: Mapping[str, Any], lines: Sequence[str] | None, function: str) -> str:
+    """The language the excerpt matcher runs under: the record's (the closed feature vocabulary, ``other`` for Go,
+    Ruby, C#, Perl, ...), and the label's -- the catalog's or the clone's -- where the record's resolves nothing
+    (a Ruby ``def`` under ``other`` has no brace body). The record keeps its own language field."""
+    language = str(record.get("language") or label.get("language") or "other")
+    declared = str(label.get("language") or "")
+    if lines and declared and declared != language and function_bounds(lines, language, function) is None:
+        return declared
+    return language
+
+
 def build_examples(
     labels: Iterable[Label | Mapping[str, Any]],
     records: Mapping[tuple[str, str, str, str, str], Mapping[str, Any]],
@@ -165,16 +176,17 @@ def build_examples(
         count.recorded += 1
         path, _, function = candidate.partition("::")
         function = (function_of(label, str(label.get("pin_role") or "vulnerable")) if function_of else None) or function
-        language = str(record.get("language") or label.get("language") or "other")
         identities = identities_of(str(label.get("repo") or ""), path, [str(label.get("pin") or "")])
         lines = lines_of(label, str(label.get("pin_role") or "vulnerable"))
+        language = str(record.get("language") or label.get("language") or "other")
+        matcher = _excerpt_language(record, label, lines, function)
         diff = None
         if unit == "delta" and lines:
             base = lines_of(label, _counter_role(label))
-            head_span, base_span = function_bounds(lines, language, function), function_bounds(base or (), language, function)
+            head_span, base_span = function_bounds(lines, matcher, function), function_bounds(base or (), matcher, function)
             if base and head_span and base_span:
-                diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], language, identities=identities)
+                diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], matcher, identities=identities)
-        shown = excerpt(lines, language, function, bounds=bounds, identities=identities, diff=diff)
+        shown = excerpt(lines, matcher, function, bounds=bounds, identities=identities, diff=diff)
         if shown is None:
             if not lines:
                 count.unread += 1

src/openultrasast/learn/excerpt.py

@@ -14,6 +14,14 @@ otherwise tell the model the label. Both become ``<redacted>``. A delta unit add
 
 The text is normalised (trailing blanks dropped, ``\\n`` endings, one final newline) before it is hashed, so the sha
 names the content an embedding and a response cache key are of.
+
+Where a function is declared (:func:`function_bounds`): the repo-facts declaration pattern of the language first (its
+span runs to the next declaration), else the brace matcher, which knows ``name(...) {`` with any return type,
+qualifiers, generics or a parameter list spread over many lines (a prototype's ``;`` is not a body), object-literal
+methods ``name: function (`` / ``name: (a) =>``, assignments ``x.y.name = function (`` / ``= (a) =>`` / chained
+``const name = m.name = async a =>``, Perl ``sub name {`` and C# expression bodies. A file given as ``other`` or
+``unknown`` is sniffed for Perl and Ruby (:func:`resolve_language`), whose ``def``/``sub`` forms a brace matcher
+would miss or mistake for a block.
 """
 
 from __future__ import annotations
@@ -180,47 +188,123 @@ def identities_of(repo: str = "", path: str = "", commits: Iterable[str] = ()) -
 # --- spans and excerpts -----------------------------------------------------------------------------------------------
 
 
-def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
+_UNKNOWN_LANGUAGES = frozenset({"", "other", "unknown"})
-    """A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first
+_PERL_LINE = re.compile(r"^\s*(?:sub\s+\w+\s*[{(;]|use\s+strict\s*;|my\s+[$@%]\w+)")
-    line naming it before ``(`` whose ``{`` comes before any ``;``, to the matching ``}``. Comments are blanked
+_RUBY_DEF = re.compile(r"^\s*def\s+(?:self\.)?\w+[?!=]?(?:\s*\(|\s|$)")
-    before braces are counted."""
+_RUBY_END = re.compile(r"^\s*end\s*$")
-    name = re.split(r"::|\.|->", function)[-1]
+_BODY_SCAN_LINES = 40  # how far past a declaration head the body's ``{`` (or a prototype's ``;``) may be
-    if not re.fullmatch(r"[A-Za-z_]\w*", name):
+_STATEMENT_HEAD = re.compile(r"\s*(return|if|while|for|switch|else|case|throw|await|yield)\b")
-        return None
+_FUNCTION_EXPR = r"(?:async\s+)?(?:function\b\s*\*?\s*[\w$]*\s*)?(?=\(|[\w$]+\s*=>)"  # function (...) / (...) => / a =>
-    call = re.compile(r"(?<![\w.>$])" + re.escape(name) + r"\s*\(")
+
+
+def resolve_language(lines: Sequence[str], language: str) -> str:
+    """The alias-normalised language; a file given as unknown (``other``, the feature vocabulary's bucket for Go, Ruby,
+    C#, Perl, ...) is sniffed for Perl (``sub name {``, ``use strict;``) and Ruby (``def`` ... ``end``), whose
+    declaration forms a brace matcher never finds and whose ``#`` comments must be redacted."""
+    language = LANGUAGE_ALIASES.get(language, language)
+    if language in _UNKNOWN_LANGUAGES and lines:
+        if any(_PERL_LINE.match(line) for line in lines):
+            return "perl"
+        if any(_RUBY_DEF.match(line) for line in lines) and any(_RUBY_END.match(line) for line in lines):
+            return "ruby"
+    return language
+
+
+def _blank_comments(lines: Sequence[str], language: str) -> list[str]:
+    style = language if language in SLASH_COMMENTS or language in HASH_COMMENTS else "c"
     code: list[str] = []
     state: str | None = None
     for line in lines:
-        ranges, state = _comment_ranges(line, state, language if language in SLASH_COMMENTS else "c")
+        ranges, state = _comment_ranges(line, state, style)
         for start, end in reversed(ranges):
             line = line[:start] + " " * (end - start) + line[end:]
         code.append(line)
-    for i, line in enumerate(code):
+    return code
-        found = call.search(line)
+
-        if found is None or re.match(r"\s*(return|if|while|for|switch|else)\b", line):
+
-            continue
+def _declaration_heads(name: str) -> re.Pattern[str]:
-        tail = "\n".join([line[found.end() :], *code[i + 1 : i + 8]])
+    """Where a brace language may declare ``name`` (the body follows): ``name(`` (a definition -- or a call, told
-        brace, semi = tail.find("{"), tail.find(";")
+    apart by :func:`_body_start`); Perl ``sub name {``; an object-literal method ``name: function (`` /
-        if brace < 0 or (0 <= semi < brace):
+    ``name: (a) =>``; an assignment ``x.y.name = function (`` / ``= (a) =>`` / ``const name = m.name = async a =>``
-            continue
+    (any chain). ``$`` is an identifier character (``Glance$serveRequest``)."""
-        depth, opened = 0, False
+    quoted = re.escape(name)
-        for j in range(i, len(code)):
+    return re.compile(
-            for c in line[found.end() :] if j == i else code[j]:
+        rf"(?<![\w.>$]){quoted}\s*(?:<[^<>(){{}};]*>\s*)?(?=\()"
+        rf"|^\s*sub\s+{quoted}\s*(?=[{{(;])"
+        rf"|(?<![\w$]){quoted}\s*:\s*{_FUNCTION_EXPR}"
+        rf"|(?<![\w$]){quoted}\s*=\s*(?:[\w$.\[\]'\"]+\s*=\s*)*{_FUNCTION_EXPR}"
+    )
+
+
+def _body_start(code: Sequence[str], line: int, column: int) -> tuple[str, int, int] | None:
+    """Scan from a declaration head for its body: ``("brace", line, column)`` of the ``{`` at parenthesis depth 0,
+    ``("expression", line, column)`` of the ``;`` or ``,`` ending an arrow function's expression body, ``None`` for a
+    prototype (``;`` before any body), a call (a ``)`` closing an outer parenthesis, a ``}``) or no body within
+    :data:`_BODY_SCAN_LINES`. Parentheses are tracked so ``f(a, function () {})`` is not a definition of ``f``."""
+    depth, arrow = 0, False
+    for j in range(line, min(line + _BODY_SCAN_LINES, len(code))):
+        text = code[j]
+        k = column if j == line else 0
+        while k < len(text):
+            c = text[k]
+            if c == "(":
+                depth += 1
+            elif c == ")":
+                depth -= 1
+                if depth < 0:
+                    return None
+            elif depth == 0:
                 if c == "{":
-                    depth, opened = depth + 1, True
+                    return ("brace", j, k)
-                elif c == "}" and opened:
+                if arrow and c in ";,":
+                    return ("expression", j, k)
+                if c in "};":  # a statement ends (a call, a prototype) before any body opened
+                    return None
+                if text.startswith("=>", k):
+                    arrow = True
+                    k += 1
+            k += 1
+    return None
+
+
+def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
+    """A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first
+    declaration head (:func:`_declaration_heads`) whose body :func:`_body_start` finds, to the matching ``}`` (or the
+    end of an expression body). Comments are blanked before anything is matched or counted."""
+    name = re.split(r"::|\.|->", function)[-1]
+    if not re.fullmatch(r"[A-Za-z_$][\w$]*", name):
+        return None
+    heads = _declaration_heads(name)
+    code = _blank_comments(lines, language)
+    for i, line in enumerate(code):
+        if _STATEMENT_HEAD.match(line):
+            continue
+        for found in heads.finditer(line):
+            body = _body_start(code, i, found.end())
+            if body is None:
+                continue
+            kind, j, k = body
+            if kind == "expression":
+                return i, j + 1
+            depth = 0
+            for row in range(j, len(code)):
+                for c in code[row][k:] if row == j else code[row]:
+                    if c == "{":
+                        depth += 1
+                    elif c == "}":
                         depth -= 1
                         if depth == 0:
-                        return i, j + 1
+                            return i, row + 1
             return i, len(code)
     return None
 
 
 def function_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
     """0-based ``[start, end)`` of the first declaration of ``function`` (the whole file for ``<global>``), or
-    ``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, ...) and
+    ``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, C#, Perl,
-    brace languages whose pattern misses the name use the brace matcher."""
+    ...) and brace languages whose pattern misses the name use the brace matcher; an unknown language is sniffed
-    language = LANGUAGE_ALIASES.get(language, language)
+    (:func:`resolve_language`)."""
+    language = resolve_language(lines, language)
     if function == GLOBAL:
         return (0, len(lines)) if lines else None
     pattern = DECLARATION.get(language)
@@ -267,6 +351,7 @@ def excerpt(
     :func:`delta_diff`) is appended under a ``diff:`` line."""
     if not lines:
         return None
+    language = resolve_language(lines, language)
     span = function_bounds(lines, language, function)
     if span is None:
         return None
@@ -300,6 +385,7 @@ def delta_diff(
 ) -> str:
     """The base->head unified diff of a function (one line of context, no file headers), redacted, at most
     ``max_lines`` lines."""
+    language = resolve_language(head, language)
     diff = [line for line in difflib.unified_diff(list(base), list(head), lineterm="", n=1)][2:]
     out: list[str] = []
     for line in diff:
@@ -314,5 +400,5 @@ def delta_diff(
 
 __all__ = [
     "CANDIDATE", "DIFF_LINES", "EXAMPLE", "REDACTED", "Bounds", "Excerpt", "delta_diff", "excerpt", "excerpt_sha",
-    "function_bounds", "identities_of", "normalise", "redact",
+    "function_bounds", "identities_of", "normalise", "redact", "resolve_language",
 ]  # fmt: skip

tests/test_learn_examples.py

@@ -158,6 +158,16 @@ def test_a_fixed_side_without_the_function_is_a_label_defect_not_an_unread_side(
     assert unread.sources["population-v1"].unread == 1 and unread.failing() == ["population-v1"]
 
 
+def test_the_excerpt_matcher_falls_back_to_the_labels_language_when_the_records_resolves_nothing() -> None:
+    """A feature record carries the closed vocabulary's language (``other`` for Ruby, Go, C#); a Python ``def`` under
+    ``other`` has no brace body, so the matcher gets the label's language instead. The record's field is kept."""
+    lab = label()
+    other = {**feature(lab), "language": "other"}
+    build = build_examples([lab], records_by_key([other]), lambda lab, role: CODE[role], sources=SOURCES)
+    assert build.summary()["examples"] == 1 and build.rows[0]["language"] == "other"  # the row keeps the vocabulary language
+    assert build.sources["population-v1"].no_excerpt == 0
+
+
 def test_pair_sides_join_the_record_stored_under_their_harvest_unit_pin() -> None:
     """The harvest stores a pair side's record under its unit pin (the blob sha1 of the side's excerpt), not under
     :func:`label_pin`; ``pins`` from the harvest ``units.json`` joins them, and without it the record is not found."""

tests/test_learn_excerpt.py

@@ -136,3 +136,224 @@ def test_brace_languages_without_a_declaration_pattern() -> None:
     assert shown is not None and shown.first == 3 and shown.last == 7 and "other(void)" not in shown.text
     assert excerpt(c, "cpp", "Parser::parse_header") is not None
     assert excerpt(["int f(void);", "int g(void) { return f(); }"], "c", "f") is None  # a prototype is not a body
+
+
+# --- declaration forms the brace matcher must know (each form once; a call or prototype is never a declaration) -------
+
+
+def _span(code: list[str], language: str, function: str) -> tuple[int, int] | None:
+    found = excerpt(code, language, function)
+    return None if found is None else (found.first, found.last)
+
+
+def test_js_object_literal_methods() -> None:
+    js = [
+        "module.exports = {",
+        "  getInfo: function () {",
+        "    return this.exec('pdfinfo ' + this.path)",
+        "  },",
+        "  ps: function(pid, options, done) {",
+        "    var cmd = 'ps -o pcpu,rss -p ' + pid",
+        "    exec(cmd, done)",
+        "  },",
+        "  performAction: function anonymous(yytext, yyleng) {",
+        "    return eval(yytext)",
+        "  },",
+        "  short: (a) => a + 1,",
+        "  use: function () { return this.getInfo().then(function (info) { return info }) },",
+        "}",
+    ]
+    assert _span(js, "javascript", "getInfo") == (2, 4)
+    assert _span(js, "javascript", "ps") == (5, 8)
+    assert _span(js, "javascript", "performAction") == (9, 11)  # the key, not the expression's own name
+    assert _span(js, "javascript", "short") == (12, 12)  # an expression body ends at its comma
+    assert _span(js, "javascript", "then") is None  # a call inside a body is not a declaration
+
+
+def test_js_property_assignment_functions_and_arrows() -> None:
+    js = [
+        "module.exports.ConfigureFilePath = (Options, FilePath) => {",
+        "  return path.join(Options.root, FilePath)",
+        "}",
+        "const listProcessesOnPort = module.exports.listProcessesOnPort = async port => {",
+        "  return exec(`lsof -i :${port}`)",
+        "}",
+        "Glance.prototype.serveRequest = function Glance$serveRequest (req, res) {",
+        "  fs.createReadStream(req.url).pipe(res)",
+        "}",
+        "this.getDaqValue = function (tagid, fromts, tots) {",
+        "  return conn.query('SELECT ' + tagid)",
+        "}",
+        "const total = sum(prices)",
+    ]
+    assert _span(js, "javascript", "ConfigureFilePath") == (1, 3)
+    assert _span(js, "javascript", "listProcessesOnPort") == (4, 6)
+    assert _span(js, "javascript", "Glance$serveRequest") == (7, 9)  # `$` is an identifier character
+    assert _span(js, "javascript", "serveRequest") == (7, 9)
+    assert _span(js, "javascript", "getDaqValue") == (10, 12)
+    assert _span(js, "javascript", "sum") is None  # an assignment of a call result declares nothing
+
+
+def test_ts_class_methods_with_modifiers_and_multiline_parameters() -> None:
+    ts = [
+        "export class Client {",
+        "  private retries = 3",
+        "  private async fetchRetry(",
+        "    url: string,",
+        "    options: any = {},",
+        "  ): Promise<Response> {",
+        "    return fetch(url, options)",
+        "  }",
+        "  handle = async (req: Request): Promise<void> => {",
+        "    await this.fetchRetry(req.url)",
+        "  }",
+        "}",
+    ]
+    assert _span(ts, "typescript", "fetchRetry") == (3, 8)  # not the `{}` default value inside the parameter list
+    assert _span(ts, "typescript", "handle") == (9, 11)
+
+
+def test_perl_sub_under_an_unknown_language() -> None:
+    perl = [
+        "#!/usr/bin/perl",
+        "use strict;",
+        "sub link_hash_cert;",  # a forward declaration has no body
+        "# CVE-2022-1292: shell injection fixed",
+        "sub link_hash_cert {",
+        "    my $fname = $_[0];",
+        "    system(\"openssl x509 -in '$fname'\");",
+        "}",
+        "sub hash_dir { opendir(my $dh, $_[0]) or die; }",
+    ]
+    assert _span(perl, "unknown", "link_hash_cert") == (5, 8)
+    assert _span(perl, "other", "hash_dir") == (9, 9)
+    shown = excerpt(perl, "unknown", "link_hash_cert", focus=(4,))
+    assert shown is not None and "CVE-2022" not in shown.text  # `#` comments are redacted once the language is known
+
+
+def test_c_declaration_with_a_long_parameter_list_and_qualifiers() -> None:
+    c = [
+        "CURLcode Curl_add_custom_headers(struct Curl_easy *data, bool is_connect, void *req);",
+        "",
+        "CURLcode Curl_add_custom_headers(struct Curl_easy *data,",
+        "                                 bool is_connect,",
+        "#ifndef USE_HYPER",
+        "                                 struct dynbuf *req",
+        "#else",
+        "                                 void *req",
+        "#endif",
+        "  )",
+        "{",
+        "  return CURLE_OK;",
+        "}",
+        "std::string Parser::header(int index) const noexcept {",
+        "  return headers[index];",
+        "}",
+        "Parser::Parser(int n) : size_(n), data_(nullptr) {",
+        "  data_ = alloc(n);",
+        "}",
+        "int main(void) { if (check(1)) { return 1; } else if (header(2)) { return 2; } return 0; }",
+    ]
+    assert _span(c, "c", "Curl_add_custom_headers") == (3, 13)  # the prototype is skipped, the body is 9 lines down
+    assert _span(c, "cpp", "Parser::header") == (14, 16)
+    assert _span(c, "cpp", "Parser") == (17, 19)  # an initializer list's commas are not statement ends
+    assert _span(c, "c", "check") is None  # `if (check(1)) {` is a call: its `)` closes an outer parenthesis
+
+
+def test_php_methods_with_modifiers_and_return_types() -> None:
+    php = [
+        "<?php",
+        "class Export {",
+        "    public static function &byRef(array $rows): ?array",
+        "    {",
+        "        return $rows;",
+        "    }",
+        "    final protected function render(string $name): void { echo $name; }",
+        "}",
+    ]
+    assert _span(php, "php", "byRef") == (3, 6)
+    assert _span(php, "php", "render") == (7, 8)  # a pattern language's span runs to the next declaration or the end
+
+
+def test_go_receivers_and_rust_generics() -> None:
+    go = [
+        "package main",
+        "",
+        "func (s *Server) handle(w http.ResponseWriter, r *http.Request) {",
+        "\tio.Copy(w, r.Body)",
+        "}",
+        "",
+        "func main() {}",
+    ]
+    assert _span(go, "go", "handle") == (3, 6)  # the declaration pattern: to the next `func`
+    assert _span(go, "other", "handle") == (3, 5)  # the feature vocabulary's bucket for Go: brace-matched to the body
+    rust = ["pub fn parse<T: Read>(input: &mut T) -> Result<(), Error> {", "    Ok(())", "}"]
+    assert _span(rust, "rust", "parse") == (1, 3)
+
+
+def test_ruby_defs_under_the_other_language_are_sniffed() -> None:
+    ruby = [
+        "module Kredis",
+        "  class Type",
+        "    def self.cast_value(value)",
+        "      YAML.load(value)",
+        "    end",
+        "",
+        "    def fetch_file(path)",
+        "      files.each { |f| return f if f.name == path }",
+        "      nil",
+        "    end",
+        "  end",
+        "end",
+    ]
+    assert _span(ruby, "ruby", "cast_value") == (3, 6)
+    assert _span(ruby, "other", "cast_value") == (3, 6)  # not None: the file is sniffed as Ruby
+    assert _span(ruby, "other", "fetch_file") == _span(ruby, "ruby", "fetch_file") == (7, 12)  # to the end: no later `def`
+    assert _span(ruby, "c", "fetch_file") == (7, 8)  # what a brace matcher takes under a named brace language: the block
+
+
+def test_csharp_attributes_modifiers_and_expression_bodies() -> None:
+    cs = [
+        "public class UploadController : Controller",
+        "{",
+        "    [HttpPost]",
+        "    public async Task<IActionResult> Upload(IFormFile file)",
+        "    {",
+        "        var path = Path.Combine(root, file.FileName);",
+        "        return Ok(path);",
+        "    }",
+        "    protected override string Name(int index) => names[index];",
+        "    public abstract int Size(string key);",
+        "}",
+    ]
+    assert _span(cs, "csharp", "Upload") == (4, 8)
+    assert _span(cs, "csharp", "Name") == (9, 9)  # an expression-bodied member ends at its `;`
+    assert _span(cs, "csharp", "Size") is None  # abstract: no body
+
+
+def test_java_and_statement_heads_are_not_declarations() -> None:
+    java = [
+        "public class Handler {",
+        "    @Override",
+        "    public synchronized <T> List<T> load(String name, Class<T> type)",
+        "            throws IOException, SQLException {",
+        "        return run(name, type);",
+        "    }",
+        "    void other() {",
+        '        while (load("x", null) != null) { break; }',
+        "    }",
+        "}",
+    ]
+    assert _span(java, "java", "load") == (3, 6)  # `throws A, B {`: the comma is not a statement end
+    assert _span(java, "java", "run") is None
+    assert _span(java[2:], "java", "load") == _span(java[2:], "c", "load") == (1, 4)  # pattern and brace matcher agree
+
+
+def test_resolve_language_sniffs_only_unknown_languages() -> None:
+    from openultrasast.learn.excerpt import resolve_language
+
+    assert resolve_language(["sub x {", "}"], "unknown") == "perl"
+    assert resolve_language(["def x", "end"], "other") == "ruby"
+    assert resolve_language(["def x(a):", "    return a"], "other") == "other"  # Python has no `end` lines
+    assert resolve_language(["sub x {", "}"], "python") == "python"  # a named language is never second-guessed
+    assert resolve_language([], "unknown") == "unknown"