mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(learn): excerpt matcher covers more declaration forms
The brace matcher now finds a declaration head in more forms -- name(...) with
any return type, qualifiers, generics or a parameter list spread over many
lines (a paren-aware scan replaces the 8-line lookahead; a prototype's ';' is
not a body, a ')' closing an outer parenthesis marks a call), JS object-literal
methods and property/chained assignments of function expressions and arrows,
$ as an identifier character, Perl 'sub name {', C# expression bodies -- and a
file given as 'other'/'unknown' is sniffed for Perl and Ruby so their def/sub
forms are matched by the right pattern instead of a {} block inside them. The
memory build hands the matcher the label's language where the record's closed
vocabulary language ('other' for Go, Ruby, C#) resolves nothing. One test per
form. Fixed-side negatives still require the fixed side to declare the
function; the 94 vibe-py positives-only groups stay counted.
Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
090265b247
commit
9d74839128
@@ -122,6 +122,17 @@ def _counter_role(label: Mapping[str, Any]) -> str:
|
||||
return "vulnerable" if label.get("pin_role") == "fixed" else "fixed"
|
||||
|
||||
|
||||
def _excerpt_language(record: Mapping[str, Any], label: Mapping[str, Any], lines: Sequence[str] | None, function: str) -> str:
|
||||
"""The language the excerpt matcher runs under: the record's (the closed feature vocabulary, ``other`` for Go,
|
||||
Ruby, C#, Perl, ...), and the label's -- the catalog's or the clone's -- where the record's resolves nothing
|
||||
(a Ruby ``def`` under ``other`` has no brace body). The record keeps its own language field."""
|
||||
language = str(record.get("language") or label.get("language") or "other")
|
||||
declared = str(label.get("language") or "")
|
||||
if lines and declared and declared != language and function_bounds(lines, language, function) is None:
|
||||
return declared
|
||||
return language
|
||||
|
||||
|
||||
def build_examples(
|
||||
labels: Iterable[Label | Mapping[str, Any]],
|
||||
records: Mapping[tuple[str, str, str, str, str], Mapping[str, Any]],
|
||||
@@ -165,16 +176,17 @@ def build_examples(
|
||||
count.recorded += 1
|
||||
path, _, function = candidate.partition("::")
|
||||
function = (function_of(label, str(label.get("pin_role") or "vulnerable")) if function_of else None) or function
|
||||
language = str(record.get("language") or label.get("language") or "other")
|
||||
identities = identities_of(str(label.get("repo") or ""), path, [str(label.get("pin") or "")])
|
||||
lines = lines_of(label, str(label.get("pin_role") or "vulnerable"))
|
||||
language = str(record.get("language") or label.get("language") or "other")
|
||||
matcher = _excerpt_language(record, label, lines, function)
|
||||
diff = None
|
||||
if unit == "delta" and lines:
|
||||
base = lines_of(label, _counter_role(label))
|
||||
head_span, base_span = function_bounds(lines, language, function), function_bounds(base or (), language, function)
|
||||
head_span, base_span = function_bounds(lines, matcher, function), function_bounds(base or (), matcher, function)
|
||||
if base and head_span and base_span:
|
||||
diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], language, identities=identities)
|
||||
shown = excerpt(lines, language, function, bounds=bounds, identities=identities, diff=diff)
|
||||
diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], matcher, identities=identities)
|
||||
shown = excerpt(lines, matcher, function, bounds=bounds, identities=identities, diff=diff)
|
||||
if shown is None:
|
||||
if not lines:
|
||||
count.unread += 1
|
||||
|
||||
@@ -14,6 +14,14 @@ otherwise tell the model the label. Both become ``<redacted>``. A delta unit add
|
||||
|
||||
The text is normalised (trailing blanks dropped, ``\\n`` endings, one final newline) before it is hashed, so the sha
|
||||
names the content an embedding and a response cache key are of.
|
||||
|
||||
Where a function is declared (:func:`function_bounds`): the repo-facts declaration pattern of the language first (its
|
||||
span runs to the next declaration), else the brace matcher, which knows ``name(...) {`` with any return type,
|
||||
qualifiers, generics or a parameter list spread over many lines (a prototype's ``;`` is not a body), object-literal
|
||||
methods ``name: function (`` / ``name: (a) =>``, assignments ``x.y.name = function (`` / ``= (a) =>`` / chained
|
||||
``const name = m.name = async a =>``, Perl ``sub name {`` and C# expression bodies. A file given as ``other`` or
|
||||
``unknown`` is sniffed for Perl and Ruby (:func:`resolve_language`), whose ``def``/``sub`` forms a brace matcher
|
||||
would miss or mistake for a block.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -180,47 +188,123 @@ def identities_of(repo: str = "", path: str = "", commits: Iterable[str] = ()) -
|
||||
# --- spans and excerpts -----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
"""A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first
|
||||
line naming it before ``(`` whose ``{`` comes before any ``;``, to the matching ``}``. Comments are blanked
|
||||
before braces are counted."""
|
||||
name = re.split(r"::|\.|->", function)[-1]
|
||||
if not re.fullmatch(r"[A-Za-z_]\w*", name):
|
||||
return None
|
||||
call = re.compile(r"(?<![\w.>$])" + re.escape(name) + r"\s*\(")
|
||||
_UNKNOWN_LANGUAGES = frozenset({"", "other", "unknown"})
|
||||
_PERL_LINE = re.compile(r"^\s*(?:sub\s+\w+\s*[{(;]|use\s+strict\s*;|my\s+[$@%]\w+)")
|
||||
_RUBY_DEF = re.compile(r"^\s*def\s+(?:self\.)?\w+[?!=]?(?:\s*\(|\s|$)")
|
||||
_RUBY_END = re.compile(r"^\s*end\s*$")
|
||||
_BODY_SCAN_LINES = 40 # how far past a declaration head the body's ``{`` (or a prototype's ``;``) may be
|
||||
_STATEMENT_HEAD = re.compile(r"\s*(return|if|while|for|switch|else|case|throw|await|yield)\b")
|
||||
_FUNCTION_EXPR = r"(?:async\s+)?(?:function\b\s*\*?\s*[\w$]*\s*)?(?=\(|[\w$]+\s*=>)" # function (...) / (...) => / a =>
|
||||
|
||||
|
||||
def resolve_language(lines: Sequence[str], language: str) -> str:
|
||||
"""The alias-normalised language; a file given as unknown (``other``, the feature vocabulary's bucket for Go, Ruby,
|
||||
C#, Perl, ...) is sniffed for Perl (``sub name {``, ``use strict;``) and Ruby (``def`` ... ``end``), whose
|
||||
declaration forms a brace matcher never finds and whose ``#`` comments must be redacted."""
|
||||
language = LANGUAGE_ALIASES.get(language, language)
|
||||
if language in _UNKNOWN_LANGUAGES and lines:
|
||||
if any(_PERL_LINE.match(line) for line in lines):
|
||||
return "perl"
|
||||
if any(_RUBY_DEF.match(line) for line in lines) and any(_RUBY_END.match(line) for line in lines):
|
||||
return "ruby"
|
||||
return language
|
||||
|
||||
|
||||
def _blank_comments(lines: Sequence[str], language: str) -> list[str]:
|
||||
style = language if language in SLASH_COMMENTS or language in HASH_COMMENTS else "c"
|
||||
code: list[str] = []
|
||||
state: str | None = None
|
||||
for line in lines:
|
||||
ranges, state = _comment_ranges(line, state, language if language in SLASH_COMMENTS else "c")
|
||||
ranges, state = _comment_ranges(line, state, style)
|
||||
for start, end in reversed(ranges):
|
||||
line = line[:start] + " " * (end - start) + line[end:]
|
||||
code.append(line)
|
||||
for i, line in enumerate(code):
|
||||
found = call.search(line)
|
||||
if found is None or re.match(r"\s*(return|if|while|for|switch|else)\b", line):
|
||||
continue
|
||||
tail = "\n".join([line[found.end() :], *code[i + 1 : i + 8]])
|
||||
brace, semi = tail.find("{"), tail.find(";")
|
||||
if brace < 0 or (0 <= semi < brace):
|
||||
continue
|
||||
depth, opened = 0, False
|
||||
for j in range(i, len(code)):
|
||||
for c in line[found.end() :] if j == i else code[j]:
|
||||
return code
|
||||
|
||||
|
||||
def _declaration_heads(name: str) -> re.Pattern[str]:
|
||||
"""Where a brace language may declare ``name`` (the body follows): ``name(`` (a definition -- or a call, told
|
||||
apart by :func:`_body_start`); Perl ``sub name {``; an object-literal method ``name: function (`` /
|
||||
``name: (a) =>``; an assignment ``x.y.name = function (`` / ``= (a) =>`` / ``const name = m.name = async a =>``
|
||||
(any chain). ``$`` is an identifier character (``Glance$serveRequest``)."""
|
||||
quoted = re.escape(name)
|
||||
return re.compile(
|
||||
rf"(?<![\w.>$]){quoted}\s*(?:<[^<>(){{}};]*>\s*)?(?=\()"
|
||||
rf"|^\s*sub\s+{quoted}\s*(?=[{{(;])"
|
||||
rf"|(?<![\w$]){quoted}\s*:\s*{_FUNCTION_EXPR}"
|
||||
rf"|(?<![\w$]){quoted}\s*=\s*(?:[\w$.\[\]'\"]+\s*=\s*)*{_FUNCTION_EXPR}"
|
||||
)
|
||||
|
||||
|
||||
def _body_start(code: Sequence[str], line: int, column: int) -> tuple[str, int, int] | None:
|
||||
"""Scan from a declaration head for its body: ``("brace", line, column)`` of the ``{`` at parenthesis depth 0,
|
||||
``("expression", line, column)`` of the ``;`` or ``,`` ending an arrow function's expression body, ``None`` for a
|
||||
prototype (``;`` before any body), a call (a ``)`` closing an outer parenthesis, a ``}``) or no body within
|
||||
:data:`_BODY_SCAN_LINES`. Parentheses are tracked so ``f(a, function () {})`` is not a definition of ``f``."""
|
||||
depth, arrow = 0, False
|
||||
for j in range(line, min(line + _BODY_SCAN_LINES, len(code))):
|
||||
text = code[j]
|
||||
k = column if j == line else 0
|
||||
while k < len(text):
|
||||
c = text[k]
|
||||
if c == "(":
|
||||
depth += 1
|
||||
elif c == ")":
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
return None
|
||||
elif depth == 0:
|
||||
if c == "{":
|
||||
depth, opened = depth + 1, True
|
||||
elif c == "}" and opened:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i, j + 1
|
||||
return i, len(code)
|
||||
return ("brace", j, k)
|
||||
if arrow and c in ";,":
|
||||
return ("expression", j, k)
|
||||
if c in "};": # a statement ends (a call, a prototype) before any body opened
|
||||
return None
|
||||
if text.startswith("=>", k):
|
||||
arrow = True
|
||||
k += 1
|
||||
k += 1
|
||||
return None
|
||||
|
||||
|
||||
def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
"""A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first
|
||||
declaration head (:func:`_declaration_heads`) whose body :func:`_body_start` finds, to the matching ``}`` (or the
|
||||
end of an expression body). Comments are blanked before anything is matched or counted."""
|
||||
name = re.split(r"::|\.|->", function)[-1]
|
||||
if not re.fullmatch(r"[A-Za-z_$][\w$]*", name):
|
||||
return None
|
||||
heads = _declaration_heads(name)
|
||||
code = _blank_comments(lines, language)
|
||||
for i, line in enumerate(code):
|
||||
if _STATEMENT_HEAD.match(line):
|
||||
continue
|
||||
for found in heads.finditer(line):
|
||||
body = _body_start(code, i, found.end())
|
||||
if body is None:
|
||||
continue
|
||||
kind, j, k = body
|
||||
if kind == "expression":
|
||||
return i, j + 1
|
||||
depth = 0
|
||||
for row in range(j, len(code)):
|
||||
for c in code[row][k:] if row == j else code[row]:
|
||||
if c == "{":
|
||||
depth += 1
|
||||
elif c == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i, row + 1
|
||||
return i, len(code)
|
||||
return None
|
||||
|
||||
|
||||
def function_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
"""0-based ``[start, end)`` of the first declaration of ``function`` (the whole file for ``<global>``), or
|
||||
``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, ...) and
|
||||
brace languages whose pattern misses the name use the brace matcher."""
|
||||
language = LANGUAGE_ALIASES.get(language, language)
|
||||
``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, C#, Perl,
|
||||
...) and brace languages whose pattern misses the name use the brace matcher; an unknown language is sniffed
|
||||
(:func:`resolve_language`)."""
|
||||
language = resolve_language(lines, language)
|
||||
if function == GLOBAL:
|
||||
return (0, len(lines)) if lines else None
|
||||
pattern = DECLARATION.get(language)
|
||||
@@ -267,6 +351,7 @@ def excerpt(
|
||||
:func:`delta_diff`) is appended under a ``diff:`` line."""
|
||||
if not lines:
|
||||
return None
|
||||
language = resolve_language(lines, language)
|
||||
span = function_bounds(lines, language, function)
|
||||
if span is None:
|
||||
return None
|
||||
@@ -300,6 +385,7 @@ def delta_diff(
|
||||
) -> str:
|
||||
"""The base->head unified diff of a function (one line of context, no file headers), redacted, at most
|
||||
``max_lines`` lines."""
|
||||
language = resolve_language(head, language)
|
||||
diff = [line for line in difflib.unified_diff(list(base), list(head), lineterm="", n=1)][2:]
|
||||
out: list[str] = []
|
||||
for line in diff:
|
||||
@@ -314,5 +400,5 @@ def delta_diff(
|
||||
|
||||
__all__ = [
|
||||
"CANDIDATE", "DIFF_LINES", "EXAMPLE", "REDACTED", "Bounds", "Excerpt", "delta_diff", "excerpt", "excerpt_sha",
|
||||
"function_bounds", "identities_of", "normalise", "redact",
|
||||
"function_bounds", "identities_of", "normalise", "redact", "resolve_language",
|
||||
] # fmt: skip
|
||||
|
||||
@@ -158,6 +158,16 @@ def test_a_fixed_side_without_the_function_is_a_label_defect_not_an_unread_side(
|
||||
assert unread.sources["population-v1"].unread == 1 and unread.failing() == ["population-v1"]
|
||||
|
||||
|
||||
def test_the_excerpt_matcher_falls_back_to_the_labels_language_when_the_records_resolves_nothing() -> None:
|
||||
"""A feature record carries the closed vocabulary's language (``other`` for Ruby, Go, C#); a Python ``def`` under
|
||||
``other`` has no brace body, so the matcher gets the label's language instead. The record's field is kept."""
|
||||
lab = label()
|
||||
other = {**feature(lab), "language": "other"}
|
||||
build = build_examples([lab], records_by_key([other]), lambda lab, role: CODE[role], sources=SOURCES)
|
||||
assert build.summary()["examples"] == 1 and build.rows[0]["language"] == "other" # the row keeps the vocabulary language
|
||||
assert build.sources["population-v1"].no_excerpt == 0
|
||||
|
||||
|
||||
def test_pair_sides_join_the_record_stored_under_their_harvest_unit_pin() -> None:
|
||||
"""The harvest stores a pair side's record under its unit pin (the blob sha1 of the side's excerpt), not under
|
||||
:func:`label_pin`; ``pins`` from the harvest ``units.json`` joins them, and without it the record is not found."""
|
||||
|
||||
@@ -136,3 +136,224 @@ def test_brace_languages_without_a_declaration_pattern() -> None:
|
||||
assert shown is not None and shown.first == 3 and shown.last == 7 and "other(void)" not in shown.text
|
||||
assert excerpt(c, "cpp", "Parser::parse_header") is not None
|
||||
assert excerpt(["int f(void);", "int g(void) { return f(); }"], "c", "f") is None # a prototype is not a body
|
||||
|
||||
|
||||
# --- declaration forms the brace matcher must know (each form once; a call or prototype is never a declaration) -------
|
||||
|
||||
|
||||
def _span(code: list[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
found = excerpt(code, language, function)
|
||||
return None if found is None else (found.first, found.last)
|
||||
|
||||
|
||||
def test_js_object_literal_methods() -> None:
|
||||
js = [
|
||||
"module.exports = {",
|
||||
" getInfo: function () {",
|
||||
" return this.exec('pdfinfo ' + this.path)",
|
||||
" },",
|
||||
" ps: function(pid, options, done) {",
|
||||
" var cmd = 'ps -o pcpu,rss -p ' + pid",
|
||||
" exec(cmd, done)",
|
||||
" },",
|
||||
" performAction: function anonymous(yytext, yyleng) {",
|
||||
" return eval(yytext)",
|
||||
" },",
|
||||
" short: (a) => a + 1,",
|
||||
" use: function () { return this.getInfo().then(function (info) { return info }) },",
|
||||
"}",
|
||||
]
|
||||
assert _span(js, "javascript", "getInfo") == (2, 4)
|
||||
assert _span(js, "javascript", "ps") == (5, 8)
|
||||
assert _span(js, "javascript", "performAction") == (9, 11) # the key, not the expression's own name
|
||||
assert _span(js, "javascript", "short") == (12, 12) # an expression body ends at its comma
|
||||
assert _span(js, "javascript", "then") is None # a call inside a body is not a declaration
|
||||
|
||||
|
||||
def test_js_property_assignment_functions_and_arrows() -> None:
|
||||
js = [
|
||||
"module.exports.ConfigureFilePath = (Options, FilePath) => {",
|
||||
" return path.join(Options.root, FilePath)",
|
||||
"}",
|
||||
"const listProcessesOnPort = module.exports.listProcessesOnPort = async port => {",
|
||||
" return exec(`lsof -i :${port}`)",
|
||||
"}",
|
||||
"Glance.prototype.serveRequest = function Glance$serveRequest (req, res) {",
|
||||
" fs.createReadStream(req.url).pipe(res)",
|
||||
"}",
|
||||
"this.getDaqValue = function (tagid, fromts, tots) {",
|
||||
" return conn.query('SELECT ' + tagid)",
|
||||
"}",
|
||||
"const total = sum(prices)",
|
||||
]
|
||||
assert _span(js, "javascript", "ConfigureFilePath") == (1, 3)
|
||||
assert _span(js, "javascript", "listProcessesOnPort") == (4, 6)
|
||||
assert _span(js, "javascript", "Glance$serveRequest") == (7, 9) # `$` is an identifier character
|
||||
assert _span(js, "javascript", "serveRequest") == (7, 9)
|
||||
assert _span(js, "javascript", "getDaqValue") == (10, 12)
|
||||
assert _span(js, "javascript", "sum") is None # an assignment of a call result declares nothing
|
||||
|
||||
|
||||
def test_ts_class_methods_with_modifiers_and_multiline_parameters() -> None:
|
||||
ts = [
|
||||
"export class Client {",
|
||||
" private retries = 3",
|
||||
" private async fetchRetry(",
|
||||
" url: string,",
|
||||
" options: any = {},",
|
||||
" ): Promise<Response> {",
|
||||
" return fetch(url, options)",
|
||||
" }",
|
||||
" handle = async (req: Request): Promise<void> => {",
|
||||
" await this.fetchRetry(req.url)",
|
||||
" }",
|
||||
"}",
|
||||
]
|
||||
assert _span(ts, "typescript", "fetchRetry") == (3, 8) # not the `{}` default value inside the parameter list
|
||||
assert _span(ts, "typescript", "handle") == (9, 11)
|
||||
|
||||
|
||||
def test_perl_sub_under_an_unknown_language() -> None:
|
||||
perl = [
|
||||
"#!/usr/bin/perl",
|
||||
"use strict;",
|
||||
"sub link_hash_cert;", # a forward declaration has no body
|
||||
"# CVE-2022-1292: shell injection fixed",
|
||||
"sub link_hash_cert {",
|
||||
" my $fname = $_[0];",
|
||||
" system(\"openssl x509 -in '$fname'\");",
|
||||
"}",
|
||||
"sub hash_dir { opendir(my $dh, $_[0]) or die; }",
|
||||
]
|
||||
assert _span(perl, "unknown", "link_hash_cert") == (5, 8)
|
||||
assert _span(perl, "other", "hash_dir") == (9, 9)
|
||||
shown = excerpt(perl, "unknown", "link_hash_cert", focus=(4,))
|
||||
assert shown is not None and "CVE-2022" not in shown.text # `#` comments are redacted once the language is known
|
||||
|
||||
|
||||
def test_c_declaration_with_a_long_parameter_list_and_qualifiers() -> None:
|
||||
c = [
|
||||
"CURLcode Curl_add_custom_headers(struct Curl_easy *data, bool is_connect, void *req);",
|
||||
"",
|
||||
"CURLcode Curl_add_custom_headers(struct Curl_easy *data,",
|
||||
" bool is_connect,",
|
||||
"#ifndef USE_HYPER",
|
||||
" struct dynbuf *req",
|
||||
"#else",
|
||||
" void *req",
|
||||
"#endif",
|
||||
" )",
|
||||
"{",
|
||||
" return CURLE_OK;",
|
||||
"}",
|
||||
"std::string Parser::header(int index) const noexcept {",
|
||||
" return headers[index];",
|
||||
"}",
|
||||
"Parser::Parser(int n) : size_(n), data_(nullptr) {",
|
||||
" data_ = alloc(n);",
|
||||
"}",
|
||||
"int main(void) { if (check(1)) { return 1; } else if (header(2)) { return 2; } return 0; }",
|
||||
]
|
||||
assert _span(c, "c", "Curl_add_custom_headers") == (3, 13) # the prototype is skipped, the body is 9 lines down
|
||||
assert _span(c, "cpp", "Parser::header") == (14, 16)
|
||||
assert _span(c, "cpp", "Parser") == (17, 19) # an initializer list's commas are not statement ends
|
||||
assert _span(c, "c", "check") is None # `if (check(1)) {` is a call: its `)` closes an outer parenthesis
|
||||
|
||||
|
||||
def test_php_methods_with_modifiers_and_return_types() -> None:
|
||||
php = [
|
||||
"<?php",
|
||||
"class Export {",
|
||||
" public static function &byRef(array $rows): ?array",
|
||||
" {",
|
||||
" return $rows;",
|
||||
" }",
|
||||
" final protected function render(string $name): void { echo $name; }",
|
||||
"}",
|
||||
]
|
||||
assert _span(php, "php", "byRef") == (3, 6)
|
||||
assert _span(php, "php", "render") == (7, 8) # a pattern language's span runs to the next declaration or the end
|
||||
|
||||
|
||||
def test_go_receivers_and_rust_generics() -> None:
|
||||
go = [
|
||||
"package main",
|
||||
"",
|
||||
"func (s *Server) handle(w http.ResponseWriter, r *http.Request) {",
|
||||
"\tio.Copy(w, r.Body)",
|
||||
"}",
|
||||
"",
|
||||
"func main() {}",
|
||||
]
|
||||
assert _span(go, "go", "handle") == (3, 6) # the declaration pattern: to the next `func`
|
||||
assert _span(go, "other", "handle") == (3, 5) # the feature vocabulary's bucket for Go: brace-matched to the body
|
||||
rust = ["pub fn parse<T: Read>(input: &mut T) -> Result<(), Error> {", " Ok(())", "}"]
|
||||
assert _span(rust, "rust", "parse") == (1, 3)
|
||||
|
||||
|
||||
def test_ruby_defs_under_the_other_language_are_sniffed() -> None:
|
||||
ruby = [
|
||||
"module Kredis",
|
||||
" class Type",
|
||||
" def self.cast_value(value)",
|
||||
" YAML.load(value)",
|
||||
" end",
|
||||
"",
|
||||
" def fetch_file(path)",
|
||||
" files.each { |f| return f if f.name == path }",
|
||||
" nil",
|
||||
" end",
|
||||
" end",
|
||||
"end",
|
||||
]
|
||||
assert _span(ruby, "ruby", "cast_value") == (3, 6)
|
||||
assert _span(ruby, "other", "cast_value") == (3, 6) # not None: the file is sniffed as Ruby
|
||||
assert _span(ruby, "other", "fetch_file") == _span(ruby, "ruby", "fetch_file") == (7, 12) # to the end: no later `def`
|
||||
assert _span(ruby, "c", "fetch_file") == (7, 8) # what a brace matcher takes under a named brace language: the block
|
||||
|
||||
|
||||
def test_csharp_attributes_modifiers_and_expression_bodies() -> None:
|
||||
cs = [
|
||||
"public class UploadController : Controller",
|
||||
"{",
|
||||
" [HttpPost]",
|
||||
" public async Task<IActionResult> Upload(IFormFile file)",
|
||||
" {",
|
||||
" var path = Path.Combine(root, file.FileName);",
|
||||
" return Ok(path);",
|
||||
" }",
|
||||
" protected override string Name(int index) => names[index];",
|
||||
" public abstract int Size(string key);",
|
||||
"}",
|
||||
]
|
||||
assert _span(cs, "csharp", "Upload") == (4, 8)
|
||||
assert _span(cs, "csharp", "Name") == (9, 9) # an expression-bodied member ends at its `;`
|
||||
assert _span(cs, "csharp", "Size") is None # abstract: no body
|
||||
|
||||
|
||||
def test_java_and_statement_heads_are_not_declarations() -> None:
|
||||
java = [
|
||||
"public class Handler {",
|
||||
" @Override",
|
||||
" public synchronized <T> List<T> load(String name, Class<T> type)",
|
||||
" throws IOException, SQLException {",
|
||||
" return run(name, type);",
|
||||
" }",
|
||||
" void other() {",
|
||||
' while (load("x", null) != null) { break; }',
|
||||
" }",
|
||||
"}",
|
||||
]
|
||||
assert _span(java, "java", "load") == (3, 6) # `throws A, B {`: the comma is not a statement end
|
||||
assert _span(java, "java", "run") is None
|
||||
assert _span(java[2:], "java", "load") == _span(java[2:], "c", "load") == (1, 4) # pattern and brace matcher agree
|
||||
|
||||
|
||||
def test_resolve_language_sniffs_only_unknown_languages() -> None:
|
||||
from openultrasast.learn.excerpt import resolve_language
|
||||
|
||||
assert resolve_language(["sub x {", "}"], "unknown") == "perl"
|
||||
assert resolve_language(["def x", "end"], "other") == "ruby"
|
||||
assert resolve_language(["def x(a):", " return a"], "other") == "other" # Python has no `end` lines
|
||||
assert resolve_language(["sub x {", "}"], "python") == "python" # a named language is never second-guessed
|
||||
assert resolve_language([], "unknown") == "unknown"
|
||||
|
||||
Reference in New Issue
Block a user