"""Excerpts for the decision engine (learned-decision-engine design 4.2): bounds, centring, no identities, advisory ids and security wording in comments redacted, the delta diff.""" from __future__ import annotations from openultrasast.learn.excerpt import ( CANDIDATE, EXAMPLE, REDACTED, Bounds, delta_diff, excerpt, excerpt_sha, identities_of, normalise, redact, ) PY = [ "import os", "", "def handler(request):", " # CVE-2021-1234: prevent SQL injection by quoting", " name = request.args['name']", ' """Fetch the user. Fixes GHSA-abcd-efgh-ijkm."""', " query = 'SELECT * FROM users WHERE name = ' + name # build the query", " return db.execute(query)", "", "def other():", " return 1", ] def _long_function(n: int, width: int = 10) -> list[str]: return ["def big(x):"] + [f" v{i} = x + {i}" + " " * width for i in range(n)] + [" return v0", "", "def after():", " pass"] def test_function_only_numbered_from_its_first_line() -> None: shown = excerpt(PY, "python", "handler") assert shown is not None lines = shown.text.splitlines() assert lines[0].split()[0] == "3" and "def handler" in lines[0] assert "def other" not in shown.text and shown.first == 3 and shown.last == 9 assert shown.sha == excerpt_sha(shown.text) and not shown.truncated def test_unread_file_or_unknown_function_is_none_never_empty() -> None: assert excerpt(None, "python", "handler") is None assert excerpt([], "python", "handler") is None assert excerpt(PY, "python", "missing") is None assert excerpt(PY, "cobol", "handler") is None def test_security_comments_and_advisory_ids_are_redacted() -> None: shown = excerpt(PY, "python", "handler") assert shown is not None assert "CVE-2021" not in shown.text and "GHSA" not in shown.text and "injection" not in shown.text assert "prevent" not in shown.text and REDACTED in shown.text assert "# build the query" in shown.text # an ordinary comment stays assert "query = 'SELECT * FROM users WHERE name = ' + name" in shown.text # code is never redacted def test_redaction_covers_block_comments_and_other_languages() -> None: php = ["function f($id) {", " /* sanitize to avoid XSS", " (see report) */", " echo $id; // 修复越权", "}"] out = redact(php, "php") assert "XSS" not in "\n".join(out) and "修复越权" not in out[3] and out[3].startswith(" echo $id;") assert "#" not in redact(["x = '#not a comment'"], "python")[0].replace("'#not a comment'", "") assert redact(["s = 'CVE-2020-0001'"], "python") == [f"s = '{REDACTED}'"] def test_no_path_repository_or_commit_in_the_excerpt() -> None: lines = [ "def handler(request):", " # see https://github.com/acme-corp/webapp/blob/0123456789abcdef0123456789abcdef01234567/app/views.py", " log('app/views.py')", " return request", ] ids = identities_of("https://github.com/acme-corp/webapp", "app/views.py", ["0123456789abcdef0123456789abcdef01234567"]) shown = excerpt(lines, "python", "handler", identities=ids) assert shown is not None for identity in ("acme-corp", "0123456789abcdef", "app/views.py"): assert identity not in shown.text assert "webapp" in shown.text or REDACTED in shown.text # the bare name is code-level, the owner and slug are not def test_bounds_lines_and_characters() -> None: lines = _long_function(200) shown = excerpt(lines, "python", "big", bounds=CANDIDATE) assert shown is not None and shown.truncated assert len(shown.text.splitlines()) <= CANDIDATE.lines and len(shown.text) <= CANDIDATE.chars small = excerpt(lines, "python", "big", bounds=EXAMPLE) assert small is not None and len(small.text.splitlines()) <= EXAMPLE.lines and len(small.text) <= EXAMPLE.chars wide = _long_function(60, width=120) tight = excerpt(wide, "python", "big", bounds=Bounds(80, 2000)) assert tight is not None and len(tight.text) <= 2000 and tight.truncated def test_window_centres_on_the_focus_lines() -> None: lines = _long_function(300) shown = excerpt(lines, "python", "big", bounds=Bounds(40, 100_000), focus=[200]) assert shown is not None and shown.first <= 200 <= shown.last assert abs((shown.first + shown.last) / 2 - 200) <= 1 chars = excerpt(lines, "python", "big", bounds=Bounds(80, 1200), focus=[200]) assert chars is not None and chars.first <= 200 <= chars.last def test_delta_diff_is_redacted_and_bounded() -> None: base = ["def f(x):", " return escape(x)"] head = ["def f(x):", " # CVE-2022-2222 sanitize later", " return x"] diff = delta_diff(base, head, "python") assert diff.startswith("@@") and "+ return x" in diff and "- return escape(x)" in diff assert "CVE-2022" not in diff and "sanitize" not in diff and "---" not in diff long = delta_diff([f"a{i}" for i in range(100)], [f"b{i}" for i in range(100)], "python", max_lines=40) assert len(long.splitlines()) == 40 and long.splitlines()[-1].startswith("...") shown = excerpt(head, "python", "f", diff=diff) assert shown is not None and "\ndiff:\n@@" in shown.text def test_normalised_text_names_the_content() -> None: assert normalise("a \r\nb\n\n\n") == "a\nb\n" assert excerpt_sha("a \nb") == excerpt_sha("a\nb\n") def test_brace_languages_without_a_declaration_pattern() -> None: c = [ "/* CVE-2016-0001: heap overflow fixed */", "static int", "parse_header(const char *buf, size_t len)", "{", " if (len > 4) { memcpy(out, buf, len); }", " return parse_header_inner(buf); /* not a definition */", "}", "int other(void) { return parse_header(0, 0); }", ] shown = excerpt(c, "c_cpp", "parse_header") assert shown is not None and shown.first == 3 and shown.last == 7 and "other(void)" not in shown.text assert excerpt(c, "cpp", "Parser::parse_header") is not None assert excerpt(["int f(void);", "int g(void) { return f(); }"], "c", "f") is None # a prototype is not a body # --- declaration forms the brace matcher must know (each form once; a call or prototype is never a declaration) ------- def _span(code: list[str], language: str, function: str) -> tuple[int, int] | None: found = excerpt(code, language, function) return None if found is None else (found.first, found.last) def test_js_object_literal_methods() -> None: js = [ "module.exports = {", " getInfo: function () {", " return this.exec('pdfinfo ' + this.path)", " },", " ps: function(pid, options, done) {", " var cmd = 'ps -o pcpu,rss -p ' + pid", " exec(cmd, done)", " },", " performAction: function anonymous(yytext, yyleng) {", " return eval(yytext)", " },", " short: (a) => a + 1,", " use: function () { return this.getInfo().then(function (info) { return info }) },", "}", ] assert _span(js, "javascript", "getInfo") == (2, 4) assert _span(js, "javascript", "ps") == (5, 8) assert _span(js, "javascript", "performAction") == (9, 11) # the key, not the expression's own name assert _span(js, "javascript", "short") == (12, 12) # an expression body ends at its comma assert _span(js, "javascript", "then") is None # a call inside a body is not a declaration def test_js_property_assignment_functions_and_arrows() -> None: js = [ "module.exports.ConfigureFilePath = (Options, FilePath) => {", " return path.join(Options.root, FilePath)", "}", "const listProcessesOnPort = module.exports.listProcessesOnPort = async port => {", " return exec(`lsof -i :${port}`)", "}", "Glance.prototype.serveRequest = function Glance$serveRequest (req, res) {", " fs.createReadStream(req.url).pipe(res)", "}", "this.getDaqValue = function (tagid, fromts, tots) {", " return conn.query('SELECT ' + tagid)", "}", "const total = sum(prices)", ] assert _span(js, "javascript", "ConfigureFilePath") == (1, 3) assert _span(js, "javascript", "listProcessesOnPort") == (4, 6) assert _span(js, "javascript", "Glance$serveRequest") == (7, 9) # `$` is an identifier character assert _span(js, "javascript", "serveRequest") == (7, 9) assert _span(js, "javascript", "getDaqValue") == (10, 12) assert _span(js, "javascript", "sum") is None # an assignment of a call result declares nothing def test_ts_class_methods_with_modifiers_and_multiline_parameters() -> None: ts = [ "export class Client {", " private retries = 3", " private async fetchRetry(", " url: string,", " options: any = {},", " ): Promise {", " return fetch(url, options)", " }", " handle = async (req: Request): Promise => {", " await this.fetchRetry(req.url)", " }", "}", ] assert _span(ts, "typescript", "fetchRetry") == (3, 8) # not the `{}` default value inside the parameter list assert _span(ts, "typescript", "handle") == (9, 11) def test_perl_sub_under_an_unknown_language() -> None: perl = [ "#!/usr/bin/perl", "use strict;", "sub link_hash_cert;", # a forward declaration has no body "# CVE-2022-1292: shell injection fixed", "sub link_hash_cert {", " my $fname = $_[0];", " system(\"openssl x509 -in '$fname'\");", "}", "sub hash_dir { opendir(my $dh, $_[0]) or die; }", ] assert _span(perl, "unknown", "link_hash_cert") == (5, 8) assert _span(perl, "other", "hash_dir") == (9, 9) shown = excerpt(perl, "unknown", "link_hash_cert", focus=(4,)) assert shown is not None and "CVE-2022" not in shown.text # `#` comments are redacted once the language is known def test_c_declaration_with_a_long_parameter_list_and_qualifiers() -> None: c = [ "CURLcode Curl_add_custom_headers(struct Curl_easy *data, bool is_connect, void *req);", "", "CURLcode Curl_add_custom_headers(struct Curl_easy *data,", " bool is_connect,", "#ifndef USE_HYPER", " struct dynbuf *req", "#else", " void *req", "#endif", " )", "{", " return CURLE_OK;", "}", "std::string Parser::header(int index) const noexcept {", " return headers[index];", "}", "Parser::Parser(int n) : size_(n), data_(nullptr) {", " data_ = alloc(n);", "}", "int main(void) { if (check(1)) { return 1; } else if (header(2)) { return 2; } return 0; }", ] assert _span(c, "c", "Curl_add_custom_headers") == (3, 13) # the prototype is skipped, the body is 9 lines down assert _span(c, "cpp", "Parser::header") == (14, 16) assert _span(c, "cpp", "Parser") == (17, 19) # an initializer list's commas are not statement ends assert _span(c, "c", "check") is None # `if (check(1)) {` is a call: its `)` closes an outer parenthesis def test_php_methods_with_modifiers_and_return_types() -> None: php = [ " None: go = [ "package main", "", "func (s *Server) handle(w http.ResponseWriter, r *http.Request) {", "\tio.Copy(w, r.Body)", "}", "", "func main() {}", ] assert _span(go, "go", "handle") == (3, 6) # the declaration pattern: to the next `func` assert _span(go, "other", "handle") == (3, 5) # the feature vocabulary's bucket for Go: brace-matched to the body rust = ["pub fn parse(input: &mut T) -> Result<(), Error> {", " Ok(())", "}"] assert _span(rust, "rust", "parse") == (1, 3) def test_ruby_defs_under_the_other_language_are_sniffed() -> None: ruby = [ "module Kredis", " class Type", " def self.cast_value(value)", " YAML.load(value)", " end", "", " def fetch_file(path)", " files.each { |f| return f if f.name == path }", " nil", " end", " end", "end", ] assert _span(ruby, "ruby", "cast_value") == (3, 6) assert _span(ruby, "other", "cast_value") == (3, 6) # not None: the file is sniffed as Ruby assert _span(ruby, "other", "fetch_file") == _span(ruby, "ruby", "fetch_file") == (7, 12) # to the end: no later `def` assert _span(ruby, "c", "fetch_file") == (7, 8) # what a brace matcher takes under a named brace language: the block def test_csharp_attributes_modifiers_and_expression_bodies() -> None: cs = [ "public class UploadController : Controller", "{", " [HttpPost]", " public async Task Upload(IFormFile file)", " {", " var path = Path.Combine(root, file.FileName);", " return Ok(path);", " }", " protected override string Name(int index) => names[index];", " public abstract int Size(string key);", "}", ] assert _span(cs, "csharp", "Upload") == (4, 8) assert _span(cs, "csharp", "Name") == (9, 9) # an expression-bodied member ends at its `;` assert _span(cs, "csharp", "Size") is None # abstract: no body def test_java_and_statement_heads_are_not_declarations() -> None: java = [ "public class Handler {", " @Override", " public synchronized List load(String name, Class type)", " throws IOException, SQLException {", " return run(name, type);", " }", " void other() {", ' while (load("x", null) != null) { break; }', " }", "}", ] assert _span(java, "java", "load") == (3, 6) # `throws A, B {`: the comma is not a statement end assert _span(java, "java", "run") is None assert _span(java[2:], "java", "load") == _span(java[2:], "c", "load") == (1, 4) # pattern and brace matcher agree def test_resolve_language_sniffs_only_unknown_languages() -> None: from openultrasast.learn.excerpt import resolve_language assert resolve_language(["sub x {", "}"], "unknown") == "perl" assert resolve_language(["def x", "end"], "other") == "ruby" assert resolve_language(["def x(a):", " return a"], "other") == "other" # Python has no `end` lines assert resolve_language(["sub x {", "}"], "python") == "python" # a named language is never second-guessed assert resolve_language([], "unknown") == "unknown"