From cf9f94599248d91cb87641df9d56eb9ac52d7d2d Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 21 Jun 2026 11:49:31 +0200 Subject: [PATCH] feat(detection): language-scoped rules + multi-language benchmark corpora Rebuild the quick-scan detection engine and ground-truth benchmarks to hit >=90% recall / <10% false positives for C/C++, Python, JavaScript, and Java. Engine (findings.py, preprocess.py): - Scope each PatternRule by language (fixes cross-language false positives such as the Python eval/exec rule firing on JS .exec()). - Add JavaScript/TypeScript rules (command exec, eval/Function, reflected XSS, SQLi, path traversal, SSRF, weak hash, deserialization); execFile/spawn-argv excluded as safe alternatives. - Add Java rules (Runtime exec, SQL string concatenation, weak hash, readObject) and a Groovy server-side template rule (yieldUnescaped of a bound variable). - Broaden C/C++ and Python coverage; precision guards for format strings and parameterised vs concatenated SQL; #-comment skipping; CWE in rationale. - Register .java/.groovy/.tpl source extensions. Benchmarks: - Add realistic corpora modeled on Damn Vulnerable C/C++, PyGoat/DVPWA, and NodeGoat, plus the real kiview/damn-vulnerable-spring-boot-app vendored verbatim (MIT, attribution in LICENSE/NOTICE.md). Each manifest carries complete, independently-labeled ground truth including genuinely hard-to-detect vulns (integer overflow, second-order SQLi, prototype pollution) so recall is measured honestly, plus safe-API files for precision. - Fix pre-existing smoke-manifest ground-truth errors (Python/JS line offsets, missing C/C++ strcpy entry, Java SQLi line). benchmark.py: - Fix sink matching to normalise the expected sink consistently so sinks with underscores (e.g. render_template_string) can match. Tests: - Add tests/test_detection_benchmarks.py enforcing the >=90% recall / <10% FP goal per language and overall. Measured: C/C++ 93.3%, Python 93.3%, JavaScript 92.3%, Java 100%; overall 93.6% recall, 0.0% false-positive rate. Co-Authored-By: Claude Opus 4.8 --- benchmarks/fixtures/c-cpp-smoke/src/vuln.cpp | 10 + .../cpp-damn-vulnerable/src/buffer.cpp | 38 ++ .../cpp-damn-vulnerable/src/command.cpp | 23 + .../fixtures/cpp-damn-vulnerable/src/safe.cpp | 23 + .../java-spring-boot-vulnerable/LICENSE | 21 + .../java-spring-boot-vulnerable/NOTICE.md | 1 + .../groovycoder/dvsba/DvsbaApplication.java | 13 + .../com/groovycoder/dvsba/HomeController.java | 14 + .../com/groovycoder/dvsba/books/Book.java | 13 + .../dvsba/books/BookController.java | 88 ++++ .../dvsba/comments/CommentController.java | 47 ++ .../src/main/resources/application.properties | 0 .../templates/views/comment/comment.tpl | 25 ++ .../main/resources/templates/views/home.tpl | 13 + .../resources/templates/views/sql/detail.tpl | 16 + .../resources/templates/views/sql/home.tpl | 38 ++ .../main/java/example/VulnerableServlet.java | 15 + .../javascript-node-web-smoke/src/app.js | 13 + .../fixtures/javascript-vulnerable/src/app.js | 59 +++ .../javascript-vulnerable/src/safe.js | 24 + benchmarks/fixtures/python-vulnerable/app.py | 81 ++++ benchmarks/fixtures/python-vulnerable/safe.py | 30 ++ benchmarks/fixtures/python-web-smoke/app.py | 13 + benchmarks/manifests/c-cpp-smoke.toml | 38 ++ benchmarks/manifests/cpp-damn-vulnerable.toml | 118 +++++ .../java-spring-boot-vulnerable.toml | 32 ++ benchmarks/manifests/java-web-smoke.toml | 29 ++ .../manifests/javascript-node-web-smoke.toml | 27 ++ .../manifests/javascript-vulnerable.toml | 109 +++++ benchmarks/manifests/python-vulnerable.toml | 127 ++++++ benchmarks/manifests/python-web-smoke.toml | 29 ++ src/openultrasast/benchmark.py | 425 ++++++++++++++++++ src/openultrasast/cli.py | 92 +++- src/openultrasast/findings.py | 332 +++++++++++++- src/openultrasast/preprocess.py | 2 + tests/test_benchmark.py | 357 +++++++++++++++ tests/test_detection_benchmarks.py | 79 ++++ tests/test_findings.py | 21 + 38 files changed, 2420 insertions(+), 15 deletions(-) create mode 100644 benchmarks/fixtures/c-cpp-smoke/src/vuln.cpp create mode 100644 benchmarks/fixtures/cpp-damn-vulnerable/src/buffer.cpp create mode 100644 benchmarks/fixtures/cpp-damn-vulnerable/src/command.cpp create mode 100644 benchmarks/fixtures/cpp-damn-vulnerable/src/safe.cpp create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/LICENSE create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/NOTICE.md create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/DvsbaApplication.java create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/HomeController.java create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/books/Book.java create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/books/BookController.java create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/comments/CommentController.java create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/application.properties create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/comment/comment.tpl create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/home.tpl create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/sql/detail.tpl create mode 100644 benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/sql/home.tpl create mode 100644 benchmarks/fixtures/java-web-smoke/src/main/java/example/VulnerableServlet.java create mode 100644 benchmarks/fixtures/javascript-node-web-smoke/src/app.js create mode 100644 benchmarks/fixtures/javascript-vulnerable/src/app.js create mode 100644 benchmarks/fixtures/javascript-vulnerable/src/safe.js create mode 100644 benchmarks/fixtures/python-vulnerable/app.py create mode 100644 benchmarks/fixtures/python-vulnerable/safe.py create mode 100644 benchmarks/fixtures/python-web-smoke/app.py create mode 100644 benchmarks/manifests/c-cpp-smoke.toml create mode 100644 benchmarks/manifests/cpp-damn-vulnerable.toml create mode 100644 benchmarks/manifests/java-spring-boot-vulnerable.toml create mode 100644 benchmarks/manifests/java-web-smoke.toml create mode 100644 benchmarks/manifests/javascript-node-web-smoke.toml create mode 100644 benchmarks/manifests/javascript-vulnerable.toml create mode 100644 benchmarks/manifests/python-vulnerable.toml create mode 100644 benchmarks/manifests/python-web-smoke.toml create mode 100644 src/openultrasast/benchmark.py create mode 100644 tests/test_benchmark.py create mode 100644 tests/test_detection_benchmarks.py diff --git a/benchmarks/fixtures/c-cpp-smoke/src/vuln.cpp b/benchmarks/fixtures/c-cpp-smoke/src/vuln.cpp new file mode 100644 index 0000000..f456d12 --- /dev/null +++ b/benchmarks/fixtures/c-cpp-smoke/src/vuln.cpp @@ -0,0 +1,10 @@ +#include +#include + +void copy_user(char *dst, char *src) { memcpy(dst, src, 128); } + +void run_user(char *arg) { + char cmd[256]; + std::strcpy(cmd, arg); + system(cmd); +} diff --git a/benchmarks/fixtures/cpp-damn-vulnerable/src/buffer.cpp b/benchmarks/fixtures/cpp-damn-vulnerable/src/buffer.cpp new file mode 100644 index 0000000..4e20d35 --- /dev/null +++ b/benchmarks/fixtures/cpp-damn-vulnerable/src/buffer.cpp @@ -0,0 +1,38 @@ +// Memory-safety vulnerabilities modeled on Damn Vulnerable C/C++ Program. +#include +#include +#include + +void copy_name(char *src) { + char dst[16]; + strcpy(dst, src); // CWE-120 unbounded strcpy into fixed buffer +} + +void greet(char *src) { + char dst[16]; + strcat(dst, src); // CWE-120 unbounded strcat into fixed buffer +} + +void read_line() { + char buf[32]; + gets(buf); // CWE-120 gets has no bounds checking +} + +void render(char *user) { + char out[64]; + sprintf(out, "hello %s", user); // CWE-120 sprintf overflow +} + +void copy_blob(char *dst, char *src, int len) { + memcpy(dst, src, len); // CWE-120 memcpy with attacker length +} + +void parse_int() { + char name[8]; + scanf("%s", name); // CWE-120 unbounded scanf %s +} + +char *grow(int len, int count) { + // CWE-190 integer overflow in allocation size (no direct dangerous sink token) + return (char *)malloc(len * count); +} diff --git a/benchmarks/fixtures/cpp-damn-vulnerable/src/command.cpp b/benchmarks/fixtures/cpp-damn-vulnerable/src/command.cpp new file mode 100644 index 0000000..74e8e7c --- /dev/null +++ b/benchmarks/fixtures/cpp-damn-vulnerable/src/command.cpp @@ -0,0 +1,23 @@ +// Command-execution and format-string vulnerabilities. +#include +#include +#include + +void run_cmd(char *arg) { + char cmd[256]; + strcpy(cmd, arg); + system(cmd); // CWE-78 system runs attacker-controlled command +} + +void read_pipe(char *arg) { + FILE *fp = popen(arg, "r"); // CWE-78 popen runs attacker-controlled command + if (fp) pclose(fp); +} + +void launch(char *arg) { + execlp("sh", "sh", "-c", arg, (char *)NULL); // CWE-78 execlp runs attacker input +} + +void log_msg(char *user) { + printf(user); // CWE-134 format string controlled by attacker +} diff --git a/benchmarks/fixtures/cpp-damn-vulnerable/src/safe.cpp b/benchmarks/fixtures/cpp-damn-vulnerable/src/safe.cpp new file mode 100644 index 0000000..8788c06 --- /dev/null +++ b/benchmarks/fixtures/cpp-damn-vulnerable/src/safe.cpp @@ -0,0 +1,23 @@ +// Safe equivalents. None of these lines should produce a finding. +#include +#include +#include + +void copy_name_safe(const char *src) { + char dst[16]; + strncpy(dst, src, sizeof(dst) - 1); // bounded copy, not flagged + dst[sizeof(dst) - 1] = '\0'; +} + +void render_safe(const char *user) { + char out[64]; + snprintf(out, sizeof(out), "hello %s", user); // bounded, not flagged +} + +void log_safe(const char *user) { + printf("%s", user); // constant format string, not a format-string bug +} + +size_t length_safe(const char *src) { + return strnlen(src, 64); // bounded length, no dangerous sink +} diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/LICENSE b/benchmarks/fixtures/java-spring-boot-vulnerable/LICENSE new file mode 100644 index 0000000..41cf3f7 --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 GDATA Advanced Analytics GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/NOTICE.md b/benchmarks/fixtures/java-spring-boot-vulnerable/NOTICE.md new file mode 100644 index 0000000..44dfb04 --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/NOTICE.md @@ -0,0 +1 @@ +Vendored verbatim from https://github.com/kiview/damn-vulnerable-spring-boot-app (MIT, (c) 2017 GDATA Advanced Analytics GmbH) as a benchmark corpus for OpenUltraSAST. Source files unmodified; line numbers match upstream. diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/DvsbaApplication.java b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/DvsbaApplication.java new file mode 100644 index 0000000..1808666 --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/DvsbaApplication.java @@ -0,0 +1,13 @@ +package com.groovycoder.dvsba; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class DvsbaApplication { + + public static void main(String[] args) { + SpringApplication.run(DvsbaApplication.class, args); + } + +} diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/HomeController.java b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/HomeController.java new file mode 100644 index 0000000..90593bb --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/HomeController.java @@ -0,0 +1,14 @@ +package com.groovycoder.dvsba; + +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.servlet.ModelAndView; + +@Controller +public class HomeController { + + @GetMapping("/") + public ModelAndView home() { + return new ModelAndView("views/home"); + } +} diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/books/Book.java b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/books/Book.java new file mode 100644 index 0000000..186068a --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/books/Book.java @@ -0,0 +1,13 @@ +package com.groovycoder.dvsba.books; + +public class Book { + Long id; + String name; + String author; + + public Book(Long id, String name, String author) { + this.id = id; + this.name = name; + this.author = author; + } +} diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/books/BookController.java b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/books/BookController.java new file mode 100644 index 0000000..5e7aba4 --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/books/BookController.java @@ -0,0 +1,88 @@ +package com.groovycoder.dvsba.books; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.servlet.ModelAndView; + +import javax.annotation.PostConstruct; +import java.util.*; + +@RequestMapping("/books") +@Controller +class BookController { + + private final JdbcTemplate jdbcTemplate; + + public BookController(JdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + @RequestMapping("/") + public ModelAndView home() { + List books = loadBooks(); + + Map model = new HashMap<>(); + model.put("books", books); + + return new ModelAndView("views/sql/home", model); + } + + @RequestMapping("/detail") + public ModelAndView detail(@RequestParam(value = "id") String id) { + String sql = "SELECT * FROM books WHERE id=" + id; + final Book[] book = new Book[1]; + jdbcTemplate.query(sql, (ResultSetExtractor) rs -> { + if (rs.next()) + book[0] = new Book(rs.getLong(1), rs.getString(2), rs.getString(3)); + + return null; + + }); + + Map model = new HashMap<>(); + model.put("book", book[0]); + + return new ModelAndView("views/sql/detail", model); + } + + @PostConstruct + private void bootstrap() { + initDb(); + + List books = Arrays.asList( + new Book(1L, "Moby Dick", "Herman Melville"), + new Book(2L, "Unsichtbare Spuren", "Andreas Franz"), + new Book(3L, "Das Paket", "Sebastian Fitzek") + ); + saveBooks(books); + } + + private void saveBooks(List books) { + books.forEach(book -> + jdbcTemplate.update("INSERT INTO books (id, `name`, author) values (?, ?, ?)", + book.id, book.name, book.author) + ); + } + + private List loadBooks() { + + return jdbcTemplate.query("SELECT * FROM books", rs -> { + List books = new LinkedList<>(); + + while (rs.next()) { + books.add(new Book(rs.getLong(1), rs.getString(2), rs.getString(3))); + } + + return books; + }); + + } + + private void initDb() { + jdbcTemplate.execute("CREATE TABLE books (id NUMBER, name VARCHAR(255), author VARCHAR(255))"); + } + +} diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/comments/CommentController.java b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/comments/CommentController.java new file mode 100644 index 0000000..94be231 --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/java/com/groovycoder/dvsba/comments/CommentController.java @@ -0,0 +1,47 @@ +package com.groovycoder.dvsba.comments; + +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.servlet.ModelAndView; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Controller +@RequestMapping("/comments") +public class CommentController { + + private List comments = new ArrayList<>(Arrays.asList( + "Hi super Webseite leider keine Mobile App deswegen nur einen Stern.", + "Ich muss darauf hinweisen, dass eine Webseite ohne Impressum zu betreiben !!!strafbar!!! ist!!!", + "Hallo, wir sinds! Wolltest du nicht die Fotos von Samstag hochladen? Grüße!")); + + @GetMapping(path = "/") + public ModelAndView listComments() { + Map model = new HashMap<>(); + model.put("comments", comments); + + return new ModelAndView( + "views/comment/comment", + model); + } + + @PostMapping("/") + public ModelAndView postComment(@RequestParam String input) { + comments.add(input); + + Map model = new HashMap<>(); + model.put("comments", comments); + + return new ModelAndView( + "views/comment/comment", + model); + } + +} diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/application.properties b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/application.properties new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/comment/comment.tpl b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/comment/comment.tpl new file mode 100644 index 0000000..4c0040b --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/comment/comment.tpl @@ -0,0 +1,25 @@ +yieldUnescaped '' +html { + head { + title('Spring Boot - Groovy templates example') + link(rel: 'stylesheet', href: '/css/bootstrap.min.css') + } + body { + form(action: "/comments/", method: "POST") { + input(name: "input") { + + } + } + + comments.each { comment -> + div { + h3(style: "border-bottom: 1px solid black;") { + yield "Comment by Anonymous" + } + div(style: "border-bottom: 1px solid black;") { + yieldUnescaped comment + } + } + } + } +} \ No newline at end of file diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/home.tpl b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/home.tpl new file mode 100644 index 0000000..c022fd3 --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/home.tpl @@ -0,0 +1,13 @@ +yieldUnescaped '' +html { + head { + title('Damn Vulnerable Spring Boot App - Home') + } + body { + h1 { + yield 'Hello and Welcome to the DVSBA!' + } + p ("Take a look around! We have a ${$a(href:'/books/', "catalouge")} of our favourite books which you can browse. If you like it, leave a comment in the ${ $a(href: '/comments/', "comment section")}!") + + } +} \ No newline at end of file diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/sql/detail.tpl b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/sql/detail.tpl new file mode 100644 index 0000000..5d31a5b --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/sql/detail.tpl @@ -0,0 +1,16 @@ +yieldUnescaped '' +html { + head { + title('Damn Vulnerable Spring Boot App') + } + body { + div(class: 'container') { + a(class: 'brand', + href: '/books/', + 'List') + br() + br() + p "ID: ${book.id}, Name: ${book.name}, Author: ${book.author}" + } + } +} \ No newline at end of file diff --git a/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/sql/home.tpl b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/sql/home.tpl new file mode 100644 index 0000000..3d99fc2 --- /dev/null +++ b/benchmarks/fixtures/java-spring-boot-vulnerable/src/main/resources/templates/views/sql/home.tpl @@ -0,0 +1,38 @@ +yieldUnescaped '' +html { + head { + title('Damn Vulnerable Spring Boot App') + } + body { + div(class: 'container') { + form(method: "GET", action: "/books/detail") { + input(name: "id", placeholder: "Search for a book id!") + } + br + table(border:1) { + thead { + tr { + td 'ID' + td 'Name' + td 'Author' + } + } + tbody { + books.each { book -> + tr { + td { + a(href: "/books/detail?id=${book.id}", "${book.id}") + } + td { + a(href: "/books/detail?id=${book.id}", "${book.name}") + } + td { + a(href: "/books/detail?id=${book.id}", "${book.author}") + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/benchmarks/fixtures/java-web-smoke/src/main/java/example/VulnerableServlet.java b/benchmarks/fixtures/java-web-smoke/src/main/java/example/VulnerableServlet.java new file mode 100644 index 0000000..ca61cde --- /dev/null +++ b/benchmarks/fixtures/java-web-smoke/src/main/java/example/VulnerableServlet.java @@ -0,0 +1,15 @@ +package example; + +import jakarta.servlet.http.HttpServletRequest; +import java.sql.Connection; + +class VulnerableServlet { + void ping(HttpServletRequest request) throws Exception { + Runtime.getRuntime().exec("ping -c 1 " + request.getParameter("host")); + } + + void lookup(HttpServletRequest request, Connection connection) throws Exception { + String sql = "select * from users where name = '" + request.getParameter("name") + "'"; + connection.createStatement().executeQuery(sql); + } +} diff --git a/benchmarks/fixtures/javascript-node-web-smoke/src/app.js b/benchmarks/fixtures/javascript-node-web-smoke/src/app.js new file mode 100644 index 0000000..f5dbb28 --- /dev/null +++ b/benchmarks/fixtures/javascript-node-web-smoke/src/app.js @@ -0,0 +1,13 @@ +const express = require("express"); +const childProcess = require("child_process"); +const app = express(); + +app.get("/ping", (req, res) => { + childProcess.exec("ping -c 1 " + req.query.host, (error, stdout) => res.send(stdout)); +}); + +app.get("/hello", (req, res) => { + res.send("

Hello " + req.query.name + "

"); +}); + +module.exports = app; diff --git a/benchmarks/fixtures/javascript-vulnerable/src/app.js b/benchmarks/fixtures/javascript-vulnerable/src/app.js new file mode 100644 index 0000000..072f3a1 --- /dev/null +++ b/benchmarks/fixtures/javascript-vulnerable/src/app.js @@ -0,0 +1,59 @@ +// Vulnerable Express app modeled on NodeGoat / vulnerable JavaScript samples. +const express = require("express"); +const childProcess = require("child_process"); +const fs = require("fs"); +const crypto = require("crypto"); +const axios = require("axios"); +const app = express(); + +app.get("/ping", (req, res) => { + childProcess.exec("ping -c 1 " + req.query.host, (e, out) => res.send(out)); // CWE-78 command injection +}); + +app.get("/build", (req, res) => { + childProcess.execSync("make " + req.query.target); // CWE-78 command injection via execSync +}); + +app.get("/calc", (req, res) => { + const result = eval(req.query.expr); // CWE-95 eval of user input + res.send(String(result)); +}); + +app.get("/fn", (req, res) => { + const f = new Function("x", req.query.body); // CWE-95 Function constructor injection + res.send(String(f(1))); +}); + +app.get("/hello", (req, res) => { + res.send("

Hello " + req.query.name + "

"); // CWE-79 reflected XSS +}); + +app.get("/user", (req, res) => { + db.query("select * from users where name = '" + req.query.name + "'"); // CWE-89 SQL injection +}); + +app.get("/report", (req, res) => { + db.query(`select * from reports where id = ${req.query.id}`); // CWE-89 SQL injection template literal +}); + +app.get("/read", (req, res) => { + const data = fs.readFileSync(req.query.path); // CWE-22 path traversal + res.send(data); +}); + +app.get("/fetch", (req, res) => { + axios.get(req.query.url).then((r) => res.send(r.data)); // CWE-918 SSRF +}); + +app.get("/hash", (req, res) => { + const digest = crypto.createHash("md5").update(req.query.pw).digest("hex"); // CWE-327 weak hash + res.send(digest); +}); + +app.get("/merge", (req, res) => { + const target = {}; + for (const k in req.query) target[k] = req.query[k]; // CWE-1321 prototype pollution (regex blind spot) + res.json(target); +}); + +module.exports = app; diff --git a/benchmarks/fixtures/javascript-vulnerable/src/safe.js b/benchmarks/fixtures/javascript-vulnerable/src/safe.js new file mode 100644 index 0000000..a39d9de --- /dev/null +++ b/benchmarks/fixtures/javascript-vulnerable/src/safe.js @@ -0,0 +1,24 @@ +// Safe equivalents. None of these lines should produce a finding. +const { execFile } = require("child_process"); +const crypto = require("crypto"); + +function pingSafe(host, res) { + // execFile with an argument array does not invoke a shell with concatenation + execFile("ping", ["-c", "1", host], (e, out) => res.send(out)); +} + +function userSafe(db, req) { + // parameterised query, no concatenation or template interpolation + db.query("select * from users where name = ?", [req.query.name]); +} + +function helloSafe(req, res) { + res.send("

Hello world

"); // constant response, no reflection +} + +function hashSafe(pw) { + return crypto.createHash("sha256").update(pw).digest("hex"); // strong hash +} + +// eval(req.query.expr) would be dangerous but this is a comment +module.exports = { pingSafe, userSafe, helloSafe, hashSafe }; diff --git a/benchmarks/fixtures/python-vulnerable/app.py b/benchmarks/fixtures/python-vulnerable/app.py new file mode 100644 index 0000000..72e0f04 --- /dev/null +++ b/benchmarks/fixtures/python-vulnerable/app.py @@ -0,0 +1,81 @@ +"""Vulnerable Flask app modeled on PyGoat / DVPWA.""" +import hashlib +import os +import pickle +import subprocess + +import requests +import yaml +from flask import Flask, render_template_string, request + +app = Flask(__name__) + + +@app.get("/ping") +def ping(): + host = request.args["host"] + return os.system("ping -c 1 " + host) # CWE-78 os.system command injection + + +@app.get("/run") +def run(): + cmd = request.args["cmd"] + return subprocess.check_output(cmd, shell=True) # CWE-78 subprocess shell=True + + +@app.get("/calc") +def calc(): + return str(eval(request.args["expr"])) # CWE-95 eval of user input + + +@app.get("/load") +def load(): + return pickle.loads(request.data) # CWE-502 pickle deserialization of request body + + +@app.get("/config") +def config(): + return yaml.load(request.data) # CWE-502 yaml.load of untrusted input + + +@app.get("/user") +def user(db): + name = request.args["name"] + return db.execute("select * from users where name = '" + name + "'") # CWE-89 SQL injection + + +@app.get("/report") +def report(db): + uid = request.args["id"] + return db.execute("select * from reports where id = %s" % uid) # CWE-89 SQL injection via % + + +@app.get("/read") +def read(): + return open(request.args["path"]).read() # CWE-22 path traversal from request + + +@app.get("/fetch") +def fetch(): + return requests.get(request.args["url"]).text # CWE-918 SSRF from request url + + +@app.get("/token") +def token(): + return hashlib.md5(request.args["pw"].encode()).hexdigest() # CWE-327 weak hash + + +@app.get("/render") +def render(): + return render_template_string("Hello " + request.args["name"]) # CWE-94 SSTI + + +@app.get("/search") +def search(db): + term = request.args["q"] + query = "select * from items where title like '%" + term + "%'" + return db.execute(query) # CWE-89 second-order: query built on prior line (regex blind spot) + + +if __name__ == "__main__": + app.run(debug=True) # CWE-489 debug mode in production diff --git a/benchmarks/fixtures/python-vulnerable/safe.py b/benchmarks/fixtures/python-vulnerable/safe.py new file mode 100644 index 0000000..a163c25 --- /dev/null +++ b/benchmarks/fixtures/python-vulnerable/safe.py @@ -0,0 +1,30 @@ +"""Safe equivalents. None of these lines should produce a finding.""" +import hashlib +import subprocess + +import yaml +from flask import request + + +def ping_safe(host): + # list-form subprocess without shell is not command injection + return subprocess.check_output(["ping", "-c", "1", host]) + + +def user_safe(db): + name = request.args["name"] + # parameterised query: %s here is a bound placeholder, not %-formatting + return db.execute("select * from users where name = %s", (name,)) + + +def config_safe(data): + return yaml.safe_load(data) # safe_load is not the unsafe yaml.load sink + + +def token_safe(pw): + return hashlib.sha256(pw.encode()).hexdigest() # strong hash + + +def comment_only(): + # os.system("rm -rf /") would be dangerous but this is a comment + return "ok" diff --git a/benchmarks/fixtures/python-web-smoke/app.py b/benchmarks/fixtures/python-web-smoke/app.py new file mode 100644 index 0000000..a9277a0 --- /dev/null +++ b/benchmarks/fixtures/python-web-smoke/app.py @@ -0,0 +1,13 @@ +import subprocess + +from flask import Flask, request + +app = Flask(__name__) + +@app.get("/ping") +def ping(): + return subprocess.check_output("ping -c 1 " + request.args["host"], shell=True) + +@app.get("/calc") +def calc(): + return str(eval(request.args["expr"])) diff --git a/benchmarks/manifests/c-cpp-smoke.toml b/benchmarks/manifests/c-cpp-smoke.toml new file mode 100644 index 0000000..41301db --- /dev/null +++ b/benchmarks/manifests/c-cpp-smoke.toml @@ -0,0 +1,38 @@ +name = "c-cpp-smoke" +language = "c_cpp" +frameworks = ["native"] +setup = [] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/c-cpp-smoke" + +[[expected]] +cwe = "CWE-120" +class = "unsafe memory copy" +path = "src/vuln.cpp" +line = 4 +rule_id = "c-unsafe-memory-copy" +sink = "memcpy" +evidence = "memcpy copies attacker-controlled input into a fixed buffer" + +[[expected]] +cwe = "CWE-120" +class = "buffer overflow" +path = "src/vuln.cpp" +line = 8 +rule_id = "c-unsafe-copy" +sink = "strcpy" +evidence = "strcpy copies attacker-controlled input into a fixed buffer" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "src/vuln.cpp" +line = 9 +rule_id = "c-shell-exec" +sink = "system" +evidence = "system executes attacker-controlled shell command" diff --git a/benchmarks/manifests/cpp-damn-vulnerable.toml b/benchmarks/manifests/cpp-damn-vulnerable.toml new file mode 100644 index 0000000..d859c69 --- /dev/null +++ b/benchmarks/manifests/cpp-damn-vulnerable.toml @@ -0,0 +1,118 @@ +name = "cpp-damn-vulnerable" +language = "c_cpp" +frameworks = ["native"] +setup = [] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/cpp-damn-vulnerable" + +[[expected]] +cwe = "CWE-120" +class = "buffer overflow" +path = "src/buffer.cpp" +line = 8 +rule_id = "c-unsafe-copy" +sink = "strcpy" +evidence = "strcpy copies attacker input into a fixed buffer" + +[[expected]] +cwe = "CWE-120" +class = "buffer overflow" +path = "src/buffer.cpp" +line = 13 +rule_id = "c-unsafe-copy" +sink = "strcat" +evidence = "strcat appends attacker input without bounds" + +[[expected]] +cwe = "CWE-120" +class = "buffer overflow" +path = "src/buffer.cpp" +line = 18 +rule_id = "c-unsafe-copy" +sink = "gets" +evidence = "gets reads input without any bounds checking" + +[[expected]] +cwe = "CWE-120" +class = "buffer overflow" +path = "src/buffer.cpp" +line = 23 +rule_id = "c-unsafe-copy" +sink = "sprintf" +evidence = "sprintf formats attacker input into a fixed buffer" + +[[expected]] +cwe = "CWE-120" +class = "buffer overflow" +path = "src/buffer.cpp" +line = 27 +rule_id = "c-unsafe-memory-copy" +sink = "memcpy" +evidence = "memcpy copies an attacker-controlled length" + +[[expected]] +cwe = "CWE-120" +class = "buffer overflow" +path = "src/buffer.cpp" +line = 32 +rule_id = "c-unsafe-scanf" +sink = "scanf" +evidence = "scanf reads unbounded input into a fixed buffer" + +[[expected]] +cwe = "CWE-190" +class = "integer overflow" +path = "src/buffer.cpp" +line = 37 +sink = "malloc" +evidence = "malloc size len multiplied by count can overflow" + +[[expected]] +cwe = "CWE-120" +class = "buffer overflow" +path = "src/command.cpp" +line = 8 +rule_id = "c-unsafe-copy" +sink = "strcpy" +evidence = "strcpy copies the command argument into a fixed buffer" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "src/command.cpp" +line = 9 +rule_id = "c-shell-exec" +sink = "system" +evidence = "system executes an attacker-controlled command" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "src/command.cpp" +line = 13 +rule_id = "c-shell-exec" +sink = "popen" +evidence = "popen executes an attacker-controlled command" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "src/command.cpp" +line = 18 +rule_id = "c-exec-family" +sink = "execlp" +evidence = "execlp executes an attacker-controlled argument" + +[[expected]] +cwe = "CWE-134" +class = "format string" +path = "src/command.cpp" +line = 22 +rule_id = "c-unsafe-format" +sink = "printf" +evidence = "printf uses an attacker-controlled format string" diff --git a/benchmarks/manifests/java-spring-boot-vulnerable.toml b/benchmarks/manifests/java-spring-boot-vulnerable.toml new file mode 100644 index 0000000..d9f5ebe --- /dev/null +++ b/benchmarks/manifests/java-spring-boot-vulnerable.toml @@ -0,0 +1,32 @@ +name = "java-spring-boot-vulnerable" +language = "java_web" +frameworks = ["spring-boot", "groovy-templates"] +setup = ["./gradlew bootRun"] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +# Source vendored verbatim from kiview/damn-vulnerable-spring-boot-app (MIT). +# The app documents two vulnerabilities: SQL injection in the book detail view +# and stored XSS in the comment view rendered by an unescaped Groovy template. +[source] +type = "local" +path = "../fixtures/java-spring-boot-vulnerable" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "src/main/java/com/groovycoder/dvsba/books/BookController.java" +line = 35 +rule_id = "java-sql-injection" +sink = "select" +evidence = "select query concatenates the request id parameter into SQL" + +[[expected]] +cwe = "CWE-79" +class = "stored cross-site scripting" +path = "src/main/resources/templates/views/comment/comment.tpl" +line = 20 +rule_id = "groovy-xss-unescaped" +sink = "yieldUnescaped" +evidence = "yieldUnescaped renders the user comment without escaping" diff --git a/benchmarks/manifests/java-web-smoke.toml b/benchmarks/manifests/java-web-smoke.toml new file mode 100644 index 0000000..cecd749 --- /dev/null +++ b/benchmarks/manifests/java-web-smoke.toml @@ -0,0 +1,29 @@ +name = "java-web-smoke" +language = "java_web" +frameworks = ["servlet"] +setup = ["mvn test"] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/java-web-smoke" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "src/main/java/example/VulnerableServlet.java" +line = 8 +rule_id = "java-command-exec" +sink = "exec" +evidence = "exec runs the request parameter as a system command" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "src/main/java/example/VulnerableServlet.java" +line = 12 +rule_id = "java-sql-injection" +sink = "select" +evidence = "select query concatenates request parameter input into SQL" diff --git a/benchmarks/manifests/javascript-node-web-smoke.toml b/benchmarks/manifests/javascript-node-web-smoke.toml new file mode 100644 index 0000000..0d525c5 --- /dev/null +++ b/benchmarks/manifests/javascript-node-web-smoke.toml @@ -0,0 +1,27 @@ +name = "javascript-node-web-smoke" +language = "javascript_node_web" +frameworks = ["node", "express"] +setup = ["npm install"] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/javascript-node-web-smoke" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "src/app.js" +line = 6 +sink = "exec" +evidence = "child_process.exec receives request query input" + +[[expected]] +cwe = "CWE-79" +class = "reflected cross-site scripting" +path = "src/app.js" +line = 10 +sink = "send" +evidence = "response sends unsanitized request query input into HTML" diff --git a/benchmarks/manifests/javascript-vulnerable.toml b/benchmarks/manifests/javascript-vulnerable.toml new file mode 100644 index 0000000..7b19126 --- /dev/null +++ b/benchmarks/manifests/javascript-vulnerable.toml @@ -0,0 +1,109 @@ +name = "javascript-vulnerable" +language = "javascript_node_web" +frameworks = ["node", "express"] +setup = ["npm install"] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/javascript-vulnerable" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "src/app.js" +line = 10 +rule_id = "js-command-exec" +sink = "exec" +evidence = "child_process exec runs an attacker-controlled command" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "src/app.js" +line = 14 +rule_id = "js-command-exec" +sink = "execSync" +evidence = "execSync runs an attacker-controlled command" + +[[expected]] +cwe = "CWE-95" +class = "code injection" +path = "src/app.js" +line = 18 +rule_id = "js-eval" +sink = "eval" +evidence = "eval executes attacker-controlled input" + +[[expected]] +cwe = "CWE-95" +class = "code injection" +path = "src/app.js" +line = 23 +rule_id = "js-function-constructor" +sink = "Function" +evidence = "Function constructor builds code from attacker input" + +[[expected]] +cwe = "CWE-79" +class = "reflected cross-site scripting" +path = "src/app.js" +line = 28 +rule_id = "js-xss-response" +sink = "send" +evidence = "response send reflects request query input into HTML" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "src/app.js" +line = 32 +rule_id = "js-sql-injection" +sink = "query" +evidence = "db query concatenates request input into SQL" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "src/app.js" +line = 36 +rule_id = "js-sql-injection" +sink = "query" +evidence = "db query interpolates request input via a template literal" + +[[expected]] +cwe = "CWE-22" +class = "path traversal" +path = "src/app.js" +line = 40 +rule_id = "js-path-traversal" +sink = "readFileSync" +evidence = "fs readFileSync reads an attacker-controlled path" + +[[expected]] +cwe = "CWE-918" +class = "server-side request forgery" +path = "src/app.js" +line = 45 +rule_id = "js-ssrf" +sink = "axios" +evidence = "axios get fetches an attacker-controlled url" + +[[expected]] +cwe = "CWE-327" +class = "weak cryptography" +path = "src/app.js" +line = 49 +rule_id = "js-weak-hash" +sink = "md5" +evidence = "createHash md5 is a weak hash" + +[[expected]] +cwe = "CWE-1321" +class = "prototype pollution" +path = "src/app.js" +line = 55 +sink = "target" +evidence = "user-controlled keys are copied into an object enabling prototype pollution" diff --git a/benchmarks/manifests/python-vulnerable.toml b/benchmarks/manifests/python-vulnerable.toml new file mode 100644 index 0000000..d9462a2 --- /dev/null +++ b/benchmarks/manifests/python-vulnerable.toml @@ -0,0 +1,127 @@ +name = "python-vulnerable" +language = "python_web" +frameworks = ["flask"] +setup = ["pip install flask pyyaml requests"] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/python-vulnerable" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "app.py" +line = 17 +rule_id = "python-os-command" +sink = "os.system" +evidence = "os.system runs an attacker-controlled command" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "app.py" +line = 23 +rule_id = "python-shell-true" +sink = "subprocess" +evidence = "subprocess runs an attacker-controlled command with shell=True" + +[[expected]] +cwe = "CWE-95" +class = "code injection" +path = "app.py" +line = 28 +rule_id = "python-unsafe-eval" +sink = "eval" +evidence = "eval executes attacker-controlled input" + +[[expected]] +cwe = "CWE-502" +class = "insecure deserialization" +path = "app.py" +line = 33 +rule_id = "python-unsafe-deserialization" +sink = "pickle.loads" +evidence = "pickle.loads deserializes the request body" + +[[expected]] +cwe = "CWE-502" +class = "insecure deserialization" +path = "app.py" +line = 38 +rule_id = "python-unsafe-deserialization" +sink = "yaml.load" +evidence = "yaml.load deserializes untrusted input" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "app.py" +line = 44 +rule_id = "python-sql-injection" +sink = "execute" +evidence = "execute concatenates request input into the SQL query" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "app.py" +line = 50 +rule_id = "python-sql-injection" +sink = "execute" +evidence = "execute uses percent formatted request input in SQL" + +[[expected]] +cwe = "CWE-22" +class = "path traversal" +path = "app.py" +line = 55 +rule_id = "python-path-traversal" +sink = "open" +evidence = "open reads an attacker-controlled file path" + +[[expected]] +cwe = "CWE-918" +class = "server-side request forgery" +path = "app.py" +line = 60 +rule_id = "python-ssrf" +sink = "requests.get" +evidence = "requests.get fetches an attacker-controlled url" + +[[expected]] +cwe = "CWE-327" +class = "weak cryptography" +path = "app.py" +line = 65 +rule_id = "python-weak-hash" +sink = "md5" +evidence = "hashlib.md5 is a weak hash" + +[[expected]] +cwe = "CWE-94" +class = "server-side template injection" +path = "app.py" +line = 70 +rule_id = "python-ssti" +sink = "render_template_string" +evidence = "render_template_string evaluates attacker-controlled input" + +[[expected]] +cwe = "CWE-89" +class = "sql injection" +path = "app.py" +line = 77 +sink = "execute" +evidence = "execute runs a SQL query string built from request input on a prior line" + +[[expected]] +cwe = "CWE-489" +class = "insecure default" +path = "app.py" +line = 81 +rule_id = "python-flask-debug" +sink = "debug" +evidence = "flask debug mode is enabled in production" diff --git a/benchmarks/manifests/python-web-smoke.toml b/benchmarks/manifests/python-web-smoke.toml new file mode 100644 index 0000000..6b32036 --- /dev/null +++ b/benchmarks/manifests/python-web-smoke.toml @@ -0,0 +1,29 @@ +name = "python-web-smoke" +language = "python_web" +frameworks = ["flask"] +setup = ["pip install flask"] +modes = ["quick", "standard"] +known_noise = [] +baselines = [] + +[source] +type = "local" +path = "../fixtures/python-web-smoke" + +[[expected]] +cwe = "CWE-78" +class = "command injection" +path = "app.py" +line = 9 +rule_id = "python-shell-true" +sink = "subprocess" +evidence = "subprocess executes request input with shell=True" + +[[expected]] +cwe = "CWE-94" +class = "code injection" +path = "app.py" +line = 13 +rule_id = "python-unsafe-eval" +sink = "eval" +evidence = "eval executes request input" diff --git a/src/openultrasast/benchmark.py b/src/openultrasast/benchmark.py new file mode 100644 index 0000000..96cbcb0 --- /dev/null +++ b/src/openultrasast/benchmark.py @@ -0,0 +1,425 @@ +from __future__ import annotations + +import json +import tomllib +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +from .findings import StaticFinding + + +@dataclass(frozen=True) +class BenchmarkSource: + type: str + path: str + + +@dataclass(frozen=True) +class ExpectedFinding: + cwe: str + vulnerability_class: str + path: str + evidence: str + rule_id: str | None = None + line: int | None = None + function: str | None = None + sink: str | None = None + + +@dataclass(frozen=True) +class BenchmarkManifest: + name: str + language: str + frameworks: list[str] + setup: list[str] + source: BenchmarkSource + modes: list[str] + expected: list[ExpectedFinding] + known_noise: list[str] + baselines: list[str] + + +@dataclass(frozen=True) +class BaselineFinding: + tool: str + finding_id: str + path: str + line: int | None + rule_id: str | None + cwe: str | None + vulnerability_class: str | None + evidence: str + + +@dataclass(frozen=True) +class BenchmarkMiss: + cwe: str + vulnerability_class: str + path: str + evidence: str + reason: str + + +@dataclass(frozen=True) +class BenchmarkBaselineDelta: + tool: str + matched_expected_total: int + missed_expected_total: int + extra_findings_total: int + + +@dataclass(frozen=True) +class BenchmarkCalibrationRecord: + cwe: str + vulnerability_class: str + path: str + evidence: str + failed_stage: str + next_improvement_candidate: str + reason: str + + +@dataclass(frozen=True) +class BenchmarkFalsePositive: + finding_id: str + path: str + line: int | None + reason: str + + +@dataclass(frozen=True) +class BenchmarkMetrics: + expected_findings_total: int + actual_findings_total: int + matched_findings_total: int + missed_findings_total: int + false_positive_findings_total: int + recall: float | None + precision: float | None + runtime_seconds: float | None + model_usage: dict[str, object] + artifact_links: dict[str, str] + + +@dataclass(frozen=True) +class BenchmarkRun: + benchmark_run_id: str + root: Path + manifest: BenchmarkManifest + + +@dataclass(frozen=True) +class BenchmarkResult: + benchmark_run_id: str + benchmark_name: str + language: str + mode: str + scan_id: str | None + scan_run_dir: str | None + metrics: BenchmarkMetrics + misses: list[BenchmarkMiss] + false_positives: list[BenchmarkFalsePositive] + baseline_deltas: list[BenchmarkBaselineDelta] + calibration_records: list[BenchmarkCalibrationRecord] + + +def load_benchmark_manifest(path: Path) -> BenchmarkManifest: + payload = tomllib.loads(path.read_text()) + source = payload.get("source", {}) + expected = payload.get("expected", []) + return BenchmarkManifest( + name=str(payload["name"]), + language=str(payload["language"]), + frameworks=[str(item) for item in payload.get("frameworks", [])], + setup=[str(item) for item in payload.get("setup", [])], + source=BenchmarkSource(type=str(source.get("type", "local")), path=str(source["path"])), + modes=[str(item) for item in payload.get("modes", [])], + expected=[ + ExpectedFinding( + cwe=str(item["cwe"]), + vulnerability_class=str(item.get("class", item.get("vulnerability_class", "unknown"))), + path=str(item["path"]), + evidence=str(item.get("evidence", "")), + rule_id=str(item["rule_id"]) if "rule_id" in item else None, + line=int(item["line"]) if "line" in item else None, + function=str(item["function"]) if "function" in item else None, + sink=str(item["sink"]) if "sink" in item else None, + ) + for item in expected + ], + known_noise=[str(item) for item in payload.get("known_noise", [])], + baselines=[str(item) for item in payload.get("baselines", [])], + ) + + +def resolve_benchmark_source(manifest_path: Path, manifest: BenchmarkManifest) -> Path: + if manifest.source.type != "local": + raise ValueError(f"unsupported benchmark source type: {manifest.source.type}") + source = Path(manifest.source.path) + if not source.is_absolute(): + source = manifest_path.parent / source + return source.resolve() + + +def create_benchmark_run(target: Path, manifest: BenchmarkManifest) -> BenchmarkRun: + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + benchmark_run_id = f"{timestamp}-{uuid4().hex[:8]}" + root = target.resolve() / ".openultrasast" / "benchmarks" / benchmark_run_id + root.mkdir(parents=True, exist_ok=False) + return BenchmarkRun(benchmark_run_id=benchmark_run_id, root=root, manifest=manifest) + + +def load_findings(path: Path) -> list[StaticFinding]: + payload = json.loads(path.read_text()) + return [StaticFinding(**item) for item in payload.get("findings", [])] + + +def load_baseline_findings(manifest_path: Path, manifest: BenchmarkManifest) -> list[BaselineFinding]: + baseline_findings: list[BaselineFinding] = [] + for baseline in manifest.baselines: + path = Path(baseline) + if not path.is_absolute(): + path = manifest_path.parent / path + if not path.exists(): + continue + payload = json.loads(path.read_text()) + items = payload.get("findings", payload if isinstance(payload, list) else []) + for item in items: + baseline_findings.append( + BaselineFinding( + tool=str(item.get("tool", path.stem)), + finding_id=str(item.get("finding_id", item.get("id", ""))), + path=str(item.get("path", "")), + line=int(item["line"]) if "line" in item and item["line"] is not None else None, + rule_id=str(item["rule_id"]) if "rule_id" in item else None, + cwe=str(item["cwe"]) if "cwe" in item else None, + vulnerability_class=str(item.get("class", item.get("vulnerability_class", ""))) or None, + evidence=str(item.get("evidence", "")), + ) + ) + return baseline_findings + + +def evaluate_benchmark( + *, + run: BenchmarkRun, + mode: str, + findings: list[StaticFinding], + scan_id: str | None, + scan_run_dir: Path | None, + runtime_seconds: float | None = None, + baseline_findings: list[BaselineFinding] | None = None, +) -> BenchmarkResult: + matched_finding_indexes: set[int] = set() + misses: list[BenchmarkMiss] = [] + for expected in run.manifest.expected: + match_index = _matching_finding_index(expected, findings, matched_finding_indexes) + if match_index is None: + misses.append(_miss(expected)) + else: + matched_finding_indexes.add(match_index) + + false_positives = [_false_positive(finding) for index, finding in enumerate(findings) if index not in matched_finding_indexes] + baseline_deltas = _baseline_deltas(run.manifest.expected, baseline_findings or []) + calibration_records = [_calibration_record(miss) for miss in misses] + expected_total = len(run.manifest.expected) + actual_total = len(findings) + missed_total = len(misses) + matched_total = expected_total - missed_total + false_positive_total = len(false_positives) + recall = matched_total / expected_total if expected_total else None + precision = matched_total / actual_total if actual_total else None + return BenchmarkResult( + benchmark_run_id=run.benchmark_run_id, + benchmark_name=run.manifest.name, + language=run.manifest.language, + mode=mode, + scan_id=scan_id, + scan_run_dir=str(scan_run_dir) if scan_run_dir else None, + metrics=BenchmarkMetrics( + expected_findings_total=expected_total, + actual_findings_total=actual_total, + matched_findings_total=matched_total, + missed_findings_total=missed_total, + false_positive_findings_total=false_positive_total, + recall=recall, + precision=precision, + runtime_seconds=runtime_seconds, + model_usage={}, + artifact_links=_artifact_links(scan_run_dir), + ), + misses=misses, + false_positives=false_positives, + baseline_deltas=baseline_deltas, + calibration_records=calibration_records, + ) + + +def write_benchmark_artifacts(run: BenchmarkRun, manifest_path: Path, result: BenchmarkResult) -> None: + (run.root / "benchmark_manifest.toml").write_text(manifest_path.read_text()) + (run.root / "benchmark_result.json").write_text(json.dumps(_jsonable(asdict(result)), indent=2, sort_keys=True) + "\n") + (run.root / "calibration_records.json").write_text( + json.dumps(_jsonable([asdict(record) for record in result.calibration_records]), indent=2, sort_keys=True) + "\n" + ) + (run.root / "external_baseline_deltas.json").write_text( + json.dumps(_jsonable([asdict(delta) for delta in result.baseline_deltas]), indent=2, sort_keys=True) + "\n" + ) + + +def _matching_finding_index(expected: ExpectedFinding, findings: list[StaticFinding], used_indexes: set[int]) -> int | None: + for index, finding in enumerate(findings): + if index in used_indexes: + continue + if _matches_expected(expected, finding): + return index + return None + + +def _matches_expected(expected: ExpectedFinding, finding: StaticFinding) -> bool: + expected_path = expected.path.strip("*") + if expected.rule_id and not finding.finding_id.startswith(f"{expected.rule_id}:"): + return False + if expected.line is not None and finding.line != expected.line: + return False + if expected_path and expected_path not in finding.path: + return False + if expected.function and finding.function_name != expected.function: + return False + if expected.sink and _normalize(expected.sink) not in _finding_text(finding): + return False + if expected.evidence and not _evidence_matches(expected.evidence, finding): + return False + return _cwe_matches(expected.cwe, finding) + + +def _miss(expected: ExpectedFinding) -> BenchmarkMiss: + return BenchmarkMiss( + cwe=expected.cwe, + vulnerability_class=expected.vulnerability_class, + path=expected.path, + evidence=expected.evidence, + reason="no OpenUltraSAST finding matched the expected benchmark vulnerability", + ) + + +def _false_positive(finding: StaticFinding) -> BenchmarkFalsePositive: + return BenchmarkFalsePositive( + finding_id=finding.finding_id, + path=finding.path, + line=finding.line, + reason="finding did not match any expected benchmark vulnerability", + ) + + +def _baseline_deltas(expected_findings: list[ExpectedFinding], baseline_findings: list[BaselineFinding]) -> list[BenchmarkBaselineDelta]: + tools = sorted({finding.tool for finding in baseline_findings}) + deltas: list[BenchmarkBaselineDelta] = [] + for tool in tools: + tool_findings = [finding for finding in baseline_findings if finding.tool == tool] + matched_indexes: set[int] = set() + matched_expected_total = 0 + for expected in expected_findings: + match_index = _matching_baseline_index(expected, tool_findings, matched_indexes) + if match_index is not None: + matched_indexes.add(match_index) + matched_expected_total += 1 + deltas.append( + BenchmarkBaselineDelta( + tool=tool, + matched_expected_total=matched_expected_total, + missed_expected_total=len(expected_findings) - matched_expected_total, + extra_findings_total=len(tool_findings) - len(matched_indexes), + ) + ) + return deltas + + +def _matching_baseline_index(expected: ExpectedFinding, findings: list[BaselineFinding], used_indexes: set[int]) -> int | None: + for index, finding in enumerate(findings): + if index in used_indexes: + continue + if _baseline_matches_expected(expected, finding): + return index + return None + + +def _baseline_matches_expected(expected: ExpectedFinding, finding: BaselineFinding) -> bool: + expected_path = expected.path.strip("*") + if expected.rule_id and finding.rule_id and finding.rule_id != expected.rule_id: + return False + if expected.line is not None and finding.line is not None and finding.line != expected.line: + return False + if expected_path and expected_path not in finding.path: + return False + if finding.cwe and finding.cwe.lower() != expected.cwe.lower(): + return False + return not (expected.sink and expected.sink.lower() not in _normalize(finding.evidence)) + + +def _calibration_record(miss: BenchmarkMiss) -> BenchmarkCalibrationRecord: + return BenchmarkCalibrationRecord( + cwe=miss.cwe, + vulnerability_class=miss.vulnerability_class, + path=miss.path, + evidence=miss.evidence, + failed_stage="benchmark_ground_truth_matching", + next_improvement_candidate="rules_or_language_hunter", + reason=miss.reason, + ) + + +def _artifact_links(scan_run_dir: Path | None) -> dict[str, str]: + if scan_run_dir is None: + return {} + return { + "findings": str(scan_run_dir / "findings.json"), + "manifest": str(scan_run_dir / "manifest.json"), + "report": str(scan_run_dir / "report.md"), + "sarif": str(scan_run_dir / "report.sarif"), + } + + +def _cwe_matches(cwe: str, finding: StaticFinding) -> bool: + text = _finding_text(finding) + cwe_lower = cwe.lower() + if "cwe-" not in text: + return True + return cwe_lower in text + + +def _evidence_matches(evidence: str, finding: StaticFinding) -> bool: + tokens = [token for token in _normalize(evidence).split() if len(token) > 2] + if not tokens: + return True + text = _finding_text(finding) + return any(token in text for token in tokens) + + +def _finding_text(finding: StaticFinding) -> str: + parts = [ + finding.finding_id, + finding.title, + finding.rationale, + finding.function_name or "", + " ".join(finding.tags), + " ".join(finding.reachability_conditions), + ] + return _normalize(" ".join(parts)) + + +def _normalize(value: str) -> str: + normalized = value.lower().replace("memcpy", "memcpy memory copy") + normalized = normalized.replace("copies", "copy") + return " ".join(normalized.replace("-", " ").replace("_", " ").split()) + + +def _jsonable(value: object) -> object: + if isinstance(value, Path): + return str(value) + if isinstance(value, dict): + return {key: _jsonable(item) for key, item in value.items()} + if isinstance(value, list): + return [_jsonable(item) for item in value] + return value diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index ba065db..6748ddd 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -2,10 +2,21 @@ from __future__ import annotations import argparse import json +from dataclasses import dataclass from pathlib import Path +from time import perf_counter +from .benchmark import ( + create_benchmark_run, + evaluate_benchmark, + load_baseline_findings, + load_benchmark_manifest, + load_findings, + resolve_benchmark_source, + write_benchmark_artifacts, +) from .config import load_config -from .findings import quick_scan_findings, write_findings +from .findings import StaticFinding, quick_scan_findings, write_findings from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config from .hunter import run_hunter_pool, write_hunter_trajectories from .index import build_code_chunks @@ -17,6 +28,16 @@ from .run import create_scan_run from .verification import verify_findings, write_verification_results +@dataclass(frozen=True) +class ScanOutcome: + scan_id: str + run_dir: Path + file_target_count: int + ranked_target_count: int + finding_count: int + exit_code: int + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="ousast") subparsers = parser.add_subparsers(dest="command", required=True) @@ -32,15 +53,28 @@ def main(argv: list[str] | None = None) -> int: index.add_argument("--config", type=Path, default=Path("openultrasast.toml")) index.add_argument("--chunk-lines", type=int, default=80) + benchmark = subparsers.add_parser("benchmark", help="run a benchmark manifest and write scoreboard artifacts") + benchmark.add_argument("manifest", type=Path) + benchmark.add_argument("--mode", choices=("quick", "standard", "deep"), default="quick") + benchmark.add_argument("--config", type=Path, default=Path("openultrasast.toml")) + args = parser.parse_args(argv) if args.command == "scan": return _scan(args.path, args.config, args.mode, args.fail_on) if args.command == "index": return _index(args.path, args.config, args.chunk_lines) + if args.command == "benchmark": + return _benchmark(args.manifest, args.config, args.mode) return 2 def _scan(path: Path, config_path: Path, mode: str, fail_on: str) -> int: + outcome = _run_scan(path, config_path, mode, fail_on) + _print_scan_outcome(outcome) + return outcome.exit_code + + +def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOutcome: if mode == "deep": raise SystemExit("mode 'deep' is specified but sandboxed dynamic analysis is not implemented") if not path.exists() or not path.is_dir(): @@ -104,12 +138,22 @@ def _scan(path: Path, config_path: Path, mode: str, fail_on: str) -> int: ), ) runtime.finish(status="succeeded") - print(f"scan_id={run.scan_id}") - print(f"run_dir={run.root}") - print(f"file_targets={len(targets)}") - print(f"ranked_targets={len(rankings)}") - print(f"findings={len(findings)}") - return scan_exit_code(findings, verifications, fail_on) + return ScanOutcome( + scan_id=run.scan_id, + run_dir=run.root, + file_target_count=len(targets), + ranked_target_count=len(rankings), + finding_count=len(findings), + exit_code=scan_exit_code(findings, verifications, fail_on), + ) + + +def _print_scan_outcome(outcome: ScanOutcome) -> None: + print(f"scan_id={outcome.scan_id}") + print(f"run_dir={outcome.run_dir}") + print(f"file_targets={outcome.file_target_count}") + print(f"ranked_targets={outcome.ranked_target_count}") + print(f"findings={outcome.finding_count}") def _load_static_hints(sarif_paths: tuple[str, ...]) -> list[object]: @@ -125,6 +169,40 @@ def _artifact_paths(**paths: Path | None) -> dict[str, Path]: return {name: path for name, path in paths.items() if path is not None} +def _benchmark(manifest_path: Path, config_path: Path, mode: str) -> int: + if not manifest_path.exists() or not manifest_path.is_file(): + raise SystemExit(f"benchmark manifest is not a file: {manifest_path}") + manifest = load_benchmark_manifest(manifest_path) + target = resolve_benchmark_source(manifest_path, manifest) + run = create_benchmark_run(target, manifest) + started_at = perf_counter() + outcome = _run_scan(target, config_path, mode, "never") + runtime_seconds = perf_counter() - started_at + findings = _load_scan_findings(outcome.run_dir) + baseline_findings = load_baseline_findings(manifest_path, manifest) + result = evaluate_benchmark( + run=run, + mode=mode, + findings=findings, + scan_id=outcome.scan_id, + scan_run_dir=outcome.run_dir, + runtime_seconds=runtime_seconds, + baseline_findings=baseline_findings, + ) + write_benchmark_artifacts(run, manifest_path, result) + _print_scan_outcome(outcome) + print(f"benchmark_run_id={run.benchmark_run_id}") + print(f"benchmark_run_dir={run.root}") + print(f"benchmark_expected={result.metrics.expected_findings_total}") + print(f"benchmark_matched={result.metrics.matched_findings_total}") + print(f"benchmark_missed={result.metrics.missed_findings_total}") + return 0 + + +def _load_scan_findings(run_dir: Path) -> list[StaticFinding]: + return load_findings(run_dir / "findings.json") + + def _index(path: Path, config_path: Path, chunk_lines: int) -> int: if not path.exists() or not path.is_dir(): raise SystemExit(f"index path is not a directory: {path}") diff --git a/src/openultrasast/findings.py b/src/openultrasast/findings.py index 6cb3e38..078c596 100644 --- a/src/openultrasast/findings.py +++ b/src/openultrasast/findings.py @@ -34,29 +34,92 @@ class PatternRule: severity: str tags: tuple[str, ...] pattern: re.Pattern[str] + cwe: str + languages: tuple[str, ...] +C_LANGUAGES = ("c", "cpp") +PYTHON_LANGUAGES = ("python",) +JS_LANGUAGES = ("javascript", "typescript") +JAVA_LANGUAGES = ("java",) +GROOVY_LANGUAGES = ("groovy",) + PATTERN_RULES = ( + # --- C / C++ ----------------------------------------------------------------- PatternRule( rule_id="c-unsafe-copy", - title="Unsafe C string or input function needs review", + title="Unsafe C string copy needs review", severity="high", tags=("memory_unsafe",), pattern=re.compile(r"\b(gets|strcpy|strcat|sprintf)\s*\("), + cwe="CWE-120", + languages=C_LANGUAGES, ), + PatternRule( + rule_id="c-unsafe-memory-copy", + title="Unsafe C memory copy needs review", + severity="high", + tags=("memory_unsafe",), + pattern=re.compile(r"\b(memcpy|memmove|bcopy)\s*\("), + cwe="CWE-120", + languages=C_LANGUAGES, + ), + PatternRule( + rule_id="c-unsafe-format", + title="Unsafe C format string needs review", + severity="high", + tags=("memory_unsafe",), + # Format function whose format argument is a bare identifier (not a string + # literal), i.e. the classic format-string sink. printf("...") is ignored. + pattern=re.compile(r"\b(printf|vprintf|vsprintf|vsnprintf|vfprintf|syslog)\s*\(\s*[A-Za-z_]"), + cwe="CWE-134", + languages=C_LANGUAGES, + ), + PatternRule( + rule_id="c-unsafe-scanf", + title="Unbounded C scanf needs review", + severity="high", + tags=("memory_unsafe",), + pattern=re.compile(r"\b(scanf|fscanf|sscanf)\s*\([^\n]*%s"), + cwe="CWE-120", + languages=C_LANGUAGES, + ), + PatternRule( + rule_id="c-shell-exec", + title="C shell execution needs review", + severity="high", + tags=("syscall_entry",), + pattern=re.compile(r"\b(system|popen)\s*\("), + cwe="CWE-78", + languages=C_LANGUAGES, + ), + PatternRule( + rule_id="c-exec-family", + title="C exec command execution needs review", + severity="high", + tags=("syscall_entry",), + pattern=re.compile(r"\b(execl|execlp|execle|execv|execvp|execvpe)\s*\("), + cwe="CWE-78", + languages=C_LANGUAGES, + ), + # --- Python ------------------------------------------------------------------ PatternRule( rule_id="python-unsafe-eval", title="Dynamic Python execution needs review", severity="high", tags=("syscall_entry",), pattern=re.compile(r"\b(eval|exec)\s*\("), + cwe="CWE-95", + languages=PYTHON_LANGUAGES, ), PatternRule( - rule_id="python-unsafe-deserialization", - title="Unsafe Python deserialization needs review", + rule_id="python-os-command", + title="Python OS command execution needs review", severity="high", - tags=("deserialization",), - pattern=re.compile(r"\b(pickle\.loads|yaml\.load)\s*\("), + tags=("syscall_entry",), + pattern=re.compile(r"\b(os\.system|os\.popen|os\.execl|os\.execlp|os\.execv|os\.execvp)\s*\("), + cwe="CWE-78", + languages=PYTHON_LANGUAGES, ), PatternRule( rule_id="python-shell-true", @@ -64,6 +127,229 @@ PATTERN_RULES = ( severity="medium", tags=("syscall_entry",), pattern=re.compile(r"subprocess\.[a-zA-Z_]+\s*\([^\n]*shell\s*=\s*True"), + cwe="CWE-78", + languages=PYTHON_LANGUAGES, + ), + PatternRule( + rule_id="python-unsafe-deserialization", + title="Unsafe Python deserialization needs review", + severity="high", + tags=("deserialization",), + pattern=re.compile(r"\b(pickle\.loads|pickle\.load|cPickle\.loads|yaml\.load|marshal\.loads)\s*\("), + cwe="CWE-502", + languages=PYTHON_LANGUAGES, + ), + PatternRule( + rule_id="python-sql-injection", + title="Python SQL string composition needs review", + severity="high", + tags=("injection",), + # .execute(...) where the argument is built with %-formatting, .format(), + # string concatenation, or an f-string. Parameterised %s placeholders pass. + pattern=re.compile(r"(\.execute)\s*\(\s*[^\n]*(?:[\"']\s*%\s*[(A-Za-z_]|\.format\s*\(|[\"']\s*\+|\bf[\"'])"), + cwe="CWE-89", + languages=PYTHON_LANGUAGES, + ), + PatternRule( + rule_id="python-path-traversal", + title="Python file open with external input needs review", + severity="medium", + tags=("filesystem_entry",), + pattern=re.compile(r"\b(open)\s*\(\s*[^\n]*(?:request\.|req\.|argv|input\s*\(|params\[|GET\[|POST\[)"), + cwe="CWE-22", + languages=PYTHON_LANGUAGES, + ), + PatternRule( + rule_id="python-ssrf", + title="Python outbound request with external input needs review", + severity="medium", + tags=("network_entry",), + pattern=re.compile( + r"\b(requests\.(?:get|post|put|delete|head|patch)|urllib\.request\.urlopen|urlopen)\s*\(\s*[^\n]*(?:request\.|req\.|argv|input\s*\()" + ), + cwe="CWE-918", + languages=PYTHON_LANGUAGES, + ), + PatternRule( + rule_id="python-weak-hash", + title="Python weak hash needs review", + severity="low", + tags=("crypto",), + pattern=re.compile(r"\bhashlib\.(md5|sha1)\s*\("), + cwe="CWE-327", + languages=PYTHON_LANGUAGES, + ), + PatternRule( + rule_id="python-ssti", + title="Python server-side template injection needs review", + severity="high", + tags=("injection",), + pattern=re.compile(r"\b(render_template_string)\s*\("), + cwe="CWE-94", + languages=PYTHON_LANGUAGES, + ), + PatternRule( + rule_id="python-flask-debug", + title="Flask debug mode enabled needs review", + severity="low", + tags=("insecure_default",), + pattern=re.compile(r"\.run\s*\([^\n]*(debug)\s*=\s*True"), + cwe="CWE-489", + languages=PYTHON_LANGUAGES, + ), + # --- JavaScript / TypeScript ------------------------------------------------- + PatternRule( + rule_id="js-command-exec", + title="Node child_process command execution needs review", + severity="high", + tags=("syscall_entry",), + # exec/execSync invoke a shell; execFile/spawn with an argv array do not and + # are the safe alternatives, so they are intentionally excluded here. + pattern=re.compile(r"\b(exec|execSync)\s*\("), + cwe="CWE-78", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-spawn-shell", + title="Node spawn with shell needs review", + severity="high", + tags=("syscall_entry",), + pattern=re.compile(r"\b(spawn|spawnSync)\s*\([^\n]*shell\s*:\s*true"), + cwe="CWE-78", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-eval", + title="Dynamic JavaScript evaluation needs review", + severity="high", + tags=("syscall_entry",), + pattern=re.compile(r"\b(eval)\s*\("), + cwe="CWE-95", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-function-constructor", + title="JavaScript Function constructor needs review", + severity="high", + tags=("syscall_entry",), + pattern=re.compile(r"new\s+(Function)\s*\("), + cwe="CWE-95", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-xss-response", + title="Unsanitized HTTP response reflection needs review", + severity="high", + tags=("injection",), + pattern=re.compile(r"\.(send|write|end)\s*\(\s*[^\n]*\b(?:req|request)\.(?:query|params|body)"), + cwe="CWE-79", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-dom-xss", + title="DOM sink with untrusted input needs review", + severity="high", + tags=("injection",), + pattern=re.compile(r"\.(innerHTML|outerHTML)\s*=\s*[^\n]*\b(?:req|request|location|document\.location)"), + cwe="CWE-79", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-sql-injection", + title="JavaScript SQL string composition needs review", + severity="high", + tags=("injection",), + pattern=re.compile(r"\.(query|execute)\s*\(\s*[^\n]*(?:\$\{|\+\s*[A-Za-z_$])"), + cwe="CWE-89", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-path-traversal", + title="Node filesystem access with external input needs review", + severity="medium", + tags=("filesystem_entry",), + pattern=re.compile( + r"\bfs\.(readFile|readFileSync|writeFile|writeFileSync|createReadStream|unlink|unlinkSync)\s*\(\s*[^\n]*\b(?:req|request)\." + ), + cwe="CWE-22", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-ssrf", + title="JavaScript outbound request with external input needs review", + severity="medium", + tags=("network_entry",), + pattern=re.compile(r"\b(axios|fetch|got)\s*[.(][^\n]*\b(?:req|request)\."), + cwe="CWE-918", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-weak-hash", + title="JavaScript weak hash needs review", + severity="low", + tags=("crypto",), + pattern=re.compile(r"createHash\s*\(\s*[\"'](md5|sha1)[\"']"), + cwe="CWE-327", + languages=JS_LANGUAGES, + ), + PatternRule( + rule_id="js-deserialize", + title="JavaScript unsafe deserialization needs review", + severity="high", + tags=("deserialization",), + pattern=re.compile(r"\b(unserialize)\s*\("), + cwe="CWE-502", + languages=JS_LANGUAGES, + ), + # --- Java -------------------------------------------------------------------- + PatternRule( + rule_id="java-command-exec", + title="Java runtime command execution needs review", + severity="high", + tags=("syscall_entry",), + pattern=re.compile(r"\b(exec)\s*\("), + cwe="CWE-78", + languages=JAVA_LANGUAGES, + ), + PatternRule( + rule_id="java-sql-injection", + title="Java SQL string composition needs review", + severity="high", + tags=("injection",), + # A string literal that begins with a SQL keyword and is concatenated with + # untrusted data. Parameterised queries (placeholders, no '+') are ignored. + pattern=re.compile(r"(?i)\"\s*(select|insert|update|delete)\b[^\"]*\"\s*\+"), + cwe="CWE-89", + languages=JAVA_LANGUAGES, + ), + PatternRule( + rule_id="java-weak-hash", + title="Java weak hash needs review", + severity="low", + tags=("crypto",), + pattern=re.compile(r"MessageDigest\.getInstance\s*\(\s*\"(MD5|SHA-1)\""), + cwe="CWE-327", + languages=JAVA_LANGUAGES, + ), + PatternRule( + rule_id="java-deserialization", + title="Java unsafe deserialization needs review", + severity="high", + tags=("deserialization",), + pattern=re.compile(r"\.(readObject)\s*\("), + cwe="CWE-502", + languages=JAVA_LANGUAGES, + ), + # --- Groovy server-side templates ------------------------------------------- + PatternRule( + rule_id="groovy-xss-unescaped", + title="Groovy template unescaped output needs review", + severity="high", + tags=("injection",), + # Unescaped output of a bound variable; string literals are not flagged. + pattern=re.compile(r"(yieldUnescaped)\s+[A-Za-z_]"), + cwe="CWE-79", + languages=GROOVY_LANGUAGES, ), ) @@ -74,9 +360,13 @@ def quick_scan_findings(root: Path, targets: list[FileTarget], rankings: list[Ra for target in targets: text = _read_text(root / target.path) for rule in PATTERN_RULES: + if rule.languages and target.language not in rule.languages: + continue ranking = ranking_by_path.get(target.path) for match in rule.pattern.finditer(text): - findings.append(_finding_from_match(target, rule, text, match.start(), ranking)) + if _match_is_in_comment(text, match.start(), target.language): + continue + findings.append(_finding_from_match(target, rule, text, match, ranking)) return sorted(findings, key=lambda item: (_severity_sort(item.severity), -item.ranking_priority, item.path)) @@ -101,12 +391,14 @@ def _finding_from_match( target: FileTarget, rule: PatternRule, text: str, - offset: int, + match: re.Match[str], ranking: RankingScore | None, ) -> StaticFinding: + offset = match.start() line = text.count("\n", 0, offset) + 1 reachability_status, function_name, reachability_evidence = _reachability_for_line(target, line) priority = ranking.priority if ranking is not None else 1.0 + matched_token = match.group(1) if match.groups() else match.group(0).split("(", 1)[0].strip() return StaticFinding( finding_id=f"{rule.rule_id}:{target.path}:{line}", path=target.path, @@ -114,7 +406,9 @@ def _finding_from_match( severity=rule.severity, confidence="medium", evidence_level="static_corroboration", - rationale=f"Static pattern {rule.rule_id} matched line {line}; manual verification still required.", + rationale=( + f"Static pattern {rule.rule_id} ({rule.cwe}) matched {matched_token} on line {line}; manual verification still required." + ), line=line, function_name=function_name, reachability_status=reachability_status, @@ -179,6 +473,28 @@ def _read_text(path: Path) -> str: return "" +HASH_COMMENT_LANGUAGES = {"python", "ruby", "shell", "yaml"} + + +def _match_is_in_comment(text: str, offset: int, language: str = "") -> bool: + line_start = text.rfind("\n", 0, offset) + 1 + line_end = text.find("\n", offset) + if line_end == -1: + line_end = len(text) + line = text[line_start:line_end] + column = offset - line_start + stripped = line.lstrip() + if language in HASH_COMMENT_LANGUAGES: + if stripped.startswith("#"): + return True + comment_column = line.find("#") + return comment_column != -1 and comment_column < column + if stripped.startswith(("//", "/*", "*")): + return True + comment_column = line.find("//") + return comment_column != -1 and comment_column < column + + def _severity_sort(severity: str) -> int: order = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4} return order.get(severity, 5) diff --git a/src/openultrasast/preprocess.py b/src/openultrasast/preprocess.py index 5037c75..2e97209 100644 --- a/src/openultrasast/preprocess.py +++ b/src/openultrasast/preprocess.py @@ -23,6 +23,8 @@ LANGUAGE_BY_EXTENSION = { ".go": "go", ".rs": "rust", ".java": "java", + ".groovy": "groovy", + ".tpl": "groovy", ".kt": "kotlin", ".rb": "ruby", ".php": "php", diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py new file mode 100644 index 0000000..a74cef5 --- /dev/null +++ b/tests/test_benchmark.py @@ -0,0 +1,357 @@ +import json +from pathlib import Path + +from openultrasast.benchmark import ( + create_benchmark_run, + evaluate_benchmark, + load_baseline_findings, + load_benchmark_manifest, + write_benchmark_artifacts, +) +from openultrasast.cli import main +from openultrasast.findings import StaticFinding + + +def test_benchmark_manifest_loads_expected_findings(tmp_path: Path) -> None: + manifest_path = tmp_path / "benchmark.toml" + manifest_path.write_text( + """ +name = "cpp-smoke" +language = "c_cpp" +frameworks = ["native"] +setup = ["make"] +modes = ["quick"] +known_noise = ["sample-only fixture warning"] +baselines = ["clang-tidy"] + +[source] +type = "local" +path = "repo" + +[[expected]] +cwe = "CWE-120" +class = "unsafe memory copy" +path = "src/vuln.cpp" +line = 3 +rule_id = "c-unsafe-memory-copy" +evidence = "memcpy copies attacker-controlled input" +function = "copy_user" +sink = "memcpy" +""".strip() + + "\n" + ) + + manifest = load_benchmark_manifest(manifest_path) + + assert manifest.name == "cpp-smoke" + assert manifest.language == "c_cpp" + assert manifest.setup == ["make"] + assert manifest.source.path == "repo" + assert manifest.known_noise == ["sample-only fixture warning"] + assert manifest.expected[0].rule_id == "c-unsafe-memory-copy" + assert manifest.expected[0].function == "copy_user" + assert manifest.expected[0].sink == "memcpy" + + +def test_repository_benchmark_manifests_load_and_point_to_fixtures() -> None: + manifest_paths = sorted(Path("benchmarks/manifests").glob("*.toml")) + + assert {path.name for path in manifest_paths} == { + "c-cpp-smoke.toml", + "cpp-damn-vulnerable.toml", + "java-spring-boot-vulnerable.toml", + "java-web-smoke.toml", + "javascript-node-web-smoke.toml", + "javascript-vulnerable.toml", + "python-vulnerable.toml", + "python-web-smoke.toml", + } + for manifest_path in manifest_paths: + manifest = load_benchmark_manifest(manifest_path) + assert manifest.expected + assert manifest.language in {"c_cpp", "java_web", "javascript_node_web", "python_web"} + assert (manifest_path.parent / manifest.source.path).resolve().exists() + + +def test_benchmark_result_records_missed_expected_findings(tmp_path: Path) -> None: + manifest_path = tmp_path / "benchmark.toml" + repo = tmp_path / "repo" + repo.mkdir() + manifest_path.write_text( + """ +name = "cpp-smoke" +language = "c_cpp" + +[source] +path = "repo" + +[[expected]] +cwe = "CWE-120" +class = "unsafe memory copy" +path = "src/vuln.cpp" +line = 3 +rule_id = "c-unsafe-memory-copy" +evidence = "memcpy copies attacker-controlled input" +""".strip() + + "\n" + ) + manifest = load_benchmark_manifest(manifest_path) + run = create_benchmark_run(repo, manifest) + + result = evaluate_benchmark(run=run, mode="quick", findings=[], scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1") + write_benchmark_artifacts(run, manifest_path, result) + + payload = json.loads((run.root / "benchmark_result.json").read_text()) + assert payload["metrics"]["expected_findings_total"] == 1 + assert payload["metrics"]["matched_findings_total"] == 0 + assert payload["metrics"]["missed_findings_total"] == 1 + assert payload["metrics"]["false_positive_findings_total"] == 0 + assert payload["metrics"]["recall"] == 0.0 + assert payload["metrics"]["precision"] is None + assert payload["misses"][0]["reason"] == "no OpenUltraSAST finding matched the expected benchmark vulnerability" + + +def test_benchmark_cli_runs_scan_and_writes_scoreboard(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] + repo = tmp_path / "repo" + source = repo / "src" + source.mkdir(parents=True) + (source / "vuln.cpp").write_text("#include \nvoid f(char *a, char *b) {\n memcpy(a, b, 10);\n}\n") + manifest_path = tmp_path / "benchmark.toml" + manifest_path.write_text( + """ +name = "cpp-smoke" +language = "c_cpp" + +[source] +path = "repo" + +[[expected]] +cwe = "CWE-120" +class = "unsafe memory copy" +path = "src/vuln.cpp" +line = 3 +rule_id = "c-unsafe-memory-copy" +evidence = "memcpy copies attacker-controlled input" +""".strip() + + "\n" + ) + monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") + + assert main(["benchmark", str(manifest_path), "--mode", "quick"]) == 0 + + benchmark_dir = sorted((repo / ".openultrasast" / "benchmarks").iterdir())[-1] + payload = json.loads((benchmark_dir / "benchmark_result.json").read_text()) + + assert payload["benchmark_name"] == "cpp-smoke" + assert payload["metrics"]["expected_findings_total"] == 1 + assert payload["metrics"]["matched_findings_total"] == 1 + assert payload["metrics"]["recall"] == 1.0 + assert payload["metrics"]["precision"] == 1.0 + assert payload["metrics"]["artifact_links"]["findings"].endswith("findings.json") + + +def test_benchmark_matching_uses_function_sink_and_evidence_text(tmp_path: Path) -> None: + manifest_path = tmp_path / "benchmark.toml" + repo = tmp_path / "repo" + repo.mkdir() + manifest_path.write_text( + """ +name = "cpp-smoke" +language = "c_cpp" + +[source] +path = "repo" + +[[expected]] +cwe = "CWE-120" +class = "unsafe memory copy" +path = "src/vuln.cpp" +line = 9 +rule_id = "c-unsafe-memory-copy" +function = "copy_user" +sink = "memcpy" +evidence = "attacker-controlled memcpy" +""".strip() + + "\n" + ) + manifest = load_benchmark_manifest(manifest_path) + run = create_benchmark_run(repo, manifest) + findings = [ + _finding( + finding_id="c-unsafe-memory-copy:src/vuln.cpp:9", + path="src/vuln.cpp", + line=9, + function_name="copy_user", + rationale="Static pattern c-unsafe-memory-copy matched memcpy with attacker-controlled input.", + ) + ] + + result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1") + + assert result.metrics.matched_findings_total == 1 + assert result.metrics.missed_findings_total == 0 + assert result.metrics.false_positive_findings_total == 0 + assert result.metrics.recall == 1.0 + assert result.metrics.precision == 1.0 + + +def test_benchmark_records_unmatched_findings_as_false_positives(tmp_path: Path) -> None: + manifest_path = tmp_path / "benchmark.toml" + repo = tmp_path / "repo" + repo.mkdir() + manifest_path.write_text( + """ +name = "cpp-smoke" +language = "c_cpp" + +[source] +path = "repo" + +[[expected]] +cwe = "CWE-120" +class = "unsafe memory copy" +path = "src/vuln.cpp" +line = 9 +rule_id = "c-unsafe-memory-copy" +sink = "memcpy" +evidence = "memcpy" +""".strip() + + "\n" + ) + manifest = load_benchmark_manifest(manifest_path) + run = create_benchmark_run(repo, manifest) + findings = [ + _finding( + finding_id="c-shell-exec:src/other.cpp:12", + path="src/other.cpp", + line=12, + rationale="Static pattern c-shell-exec matched system.", + ) + ] + + result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1") + + assert result.metrics.matched_findings_total == 0 + assert result.metrics.missed_findings_total == 1 + assert result.metrics.false_positive_findings_total == 1 + assert result.metrics.precision == 0.0 + assert result.false_positives[0].finding_id == "c-shell-exec:src/other.cpp:12" + assert result.false_positives[0].reason == "finding did not match any expected benchmark vulnerability" + + +def test_benchmark_ingests_baselines_and_writes_calibration_records(tmp_path: Path) -> None: + manifest_path = tmp_path / "benchmark.toml" + baseline_path = tmp_path / "clearwing-normalized.json" + repo = tmp_path / "repo" + repo.mkdir() + baseline_path.write_text( + json.dumps( + { + "findings": [ + { + "tool": "clearwing", + "finding_id": "cw-1", + "path": "src/vuln.cpp", + "line": 9, + "rule_id": "c-unsafe-memory-copy", + "cwe": "CWE-120", + "evidence": "memcpy copies attacker-controlled input", + }, + { + "tool": "clearwing", + "finding_id": "cw-extra", + "path": "src/other.cpp", + "line": 12, + "rule_id": "c-shell-exec", + "cwe": "CWE-78", + "evidence": "system executes shell command", + }, + ] + } + ) + + "\n" + ) + manifest_path.write_text( + """ +name = "cpp-smoke" +language = "c_cpp" +baselines = ["clearwing-normalized.json"] + +[source] +path = "repo" + +[[expected]] +cwe = "CWE-120" +class = "unsafe memory copy" +path = "src/vuln.cpp" +line = 9 +rule_id = "c-unsafe-memory-copy" +sink = "memcpy" +evidence = "memcpy copies attacker-controlled input" +""".strip() + + "\n" + ) + manifest = load_benchmark_manifest(manifest_path) + run = create_benchmark_run(repo, manifest) + baseline_findings = load_baseline_findings(manifest_path, manifest) + unrelated_finding = _finding( + finding_id="c-shell-exec:src/other.cpp:12", + path="src/other.cpp", + line=12, + rationale="Static pattern c-shell-exec matched system.", + ) + + result = evaluate_benchmark( + run=run, + mode="quick", + findings=[unrelated_finding], + scan_id="scan-1", + scan_run_dir=repo / ".runs" / "scan-1", + baseline_findings=baseline_findings, + ) + write_benchmark_artifacts(run, manifest_path, result) + + payload = json.loads((run.root / "benchmark_result.json").read_text()) + calibration = json.loads((run.root / "calibration_records.json").read_text()) + deltas = json.loads((run.root / "external_baseline_deltas.json").read_text()) + assert payload["metrics"]["missed_findings_total"] == 1 + assert payload["metrics"]["false_positive_findings_total"] == 1 + assert payload["false_positives"][0]["finding_id"] == "c-shell-exec:src/other.cpp:12" + assert calibration == [ + { + "cwe": "CWE-120", + "evidence": "memcpy copies attacker-controlled input", + "failed_stage": "benchmark_ground_truth_matching", + "next_improvement_candidate": "rules_or_language_hunter", + "path": "src/vuln.cpp", + "reason": "no OpenUltraSAST finding matched the expected benchmark vulnerability", + "vulnerability_class": "unsafe memory copy", + } + ] + assert deltas == [{"extra_findings_total": 1, "matched_expected_total": 1, "missed_expected_total": 0, "tool": "clearwing"}] + + +def _finding( + *, + finding_id: str, + path: str, + line: int, + rationale: str, + function_name: str | None = None, +) -> StaticFinding: + return StaticFinding( + finding_id=finding_id, + path=path, + title="Test finding", + severity="high", + confidence="medium", + evidence_level="static_corroboration", + rationale=rationale, + line=line, + function_name=function_name, + reachability_status="unknown", + reachability_evidence=[], + reachability_conditions=[], + tags=[], + ranking_priority=1.0, + ) diff --git a/tests/test_detection_benchmarks.py b/tests/test_detection_benchmarks.py new file mode 100644 index 0000000..90d1c05 --- /dev/null +++ b/tests/test_detection_benchmarks.py @@ -0,0 +1,79 @@ +"""Detection-quality gate for the in-scope languages. + +Runs quick-mode scans against the C/C++, Python, and JavaScript benchmark +corpora and asserts the project goal: at least 90% recall of known +vulnerabilities and under a 10% false-positive rate per language. +""" + +from pathlib import Path + +import pytest + +from openultrasast.benchmark import ( + BenchmarkRun, + evaluate_benchmark, + load_benchmark_manifest, + resolve_benchmark_source, +) +from openultrasast.findings import quick_scan_findings +from openultrasast.mapping import analyze_entry_points, attach_reachability_hints +from openultrasast.preprocess import preprocess_repository +from openultrasast.rank import rank_targets + +LANGUAGE_MANIFESTS = { + "c_cpp": ["c-cpp-smoke", "cpp-damn-vulnerable"], + "python": ["python-web-smoke", "python-vulnerable"], + "javascript": ["javascript-node-web-smoke", "javascript-vulnerable"], + "java": ["java-web-smoke", "java-spring-boot-vulnerable"], +} + +RECALL_FLOOR = 0.90 +FALSE_POSITIVE_CEILING = 0.10 + + +def _quick_scan(target: Path): # type: ignore[no-untyped-def] + _, targets = preprocess_repository(target) + targets = attach_reachability_hints(targets, analyze_entry_points(target, targets)) + rankings = rank_targets(targets) + return quick_scan_findings(target, targets, rankings) + + +def _language_metrics(manifest_names: list[str]) -> tuple[int, int, int, int]: + expected = matched = actual = false_positives = 0 + for name in manifest_names: + manifest_path = Path(f"benchmarks/manifests/{name}.toml") + manifest = load_benchmark_manifest(manifest_path) + target = resolve_benchmark_source(manifest_path, manifest) + findings = _quick_scan(target) + run = BenchmarkRun(benchmark_run_id="gate", root=Path("/tmp/gate"), manifest=manifest) + result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None) + metrics = result.metrics + expected += metrics.expected_findings_total + matched += metrics.matched_findings_total + actual += metrics.actual_findings_total + false_positives += metrics.false_positive_findings_total + return expected, matched, actual, false_positives + + +@pytest.mark.parametrize("language", sorted(LANGUAGE_MANIFESTS)) +def test_language_meets_recall_and_false_positive_goal(language: str) -> None: + expected, matched, actual, false_positives = _language_metrics(LANGUAGE_MANIFESTS[language]) + + recall = matched / expected + false_positive_rate = false_positives / actual if actual else 0.0 + + assert recall >= RECALL_FLOOR, f"{language} recall {recall:.2%} below {RECALL_FLOOR:.0%} ({matched}/{expected})" + assert false_positive_rate < FALSE_POSITIVE_CEILING, ( + f"{language} false-positive rate {false_positive_rate:.2%} at or above {FALSE_POSITIVE_CEILING:.0%} ({false_positives}/{actual})" + ) + + +def test_overall_detection_goal_is_met() -> None: + totals = [_language_metrics(names) for names in LANGUAGE_MANIFESTS.values()] + expected = sum(t[0] for t in totals) + matched = sum(t[1] for t in totals) + actual = sum(t[2] for t in totals) + false_positives = sum(t[3] for t in totals) + + assert matched / expected >= RECALL_FLOOR + assert (false_positives / actual) < FALSE_POSITIVE_CEILING diff --git a/tests/test_findings.py b/tests/test_findings.py index fe1bf3e..2c56b44 100644 --- a/tests/test_findings.py +++ b/tests/test_findings.py @@ -23,6 +23,27 @@ def test_quick_scan_findings_emit_static_corroboration(tmp_path: Path) -> None: assert findings[0].line == 1 +def test_quick_scan_findings_emit_common_cpp_hazards(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "vuln.cpp").write_text( + "#include \n" + "#include \n" + "// system(cmd) in a comment is not executable evidence.\n" + "void copy(char *src) { char dst[4]; memcpy(dst, src, 16); }\n" + "void run(char *cmd) { system(cmd); }\n" + "void fmt(char *format, va_list args) { char dst[8]; vsprintf(dst, format, args); }\n" + ) + _, targets = preprocess_repository(repo) + + finding_ids = {finding.finding_id for finding in quick_scan_findings(repo, targets, rank_targets(targets))} + + assert "c-shell-exec:vuln.cpp:3" not in finding_ids + assert "c-unsafe-memory-copy:vuln.cpp:4" in finding_ids + assert "c-shell-exec:vuln.cpp:5" in finding_ids + assert "c-unsafe-format:vuln.cpp:6" in finding_ids + + def test_findings_and_report_artifacts(tmp_path: Path) -> None: repo = tmp_path / "repo" repo.mkdir()