mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 03:59:28 +00:00
feat(detection): language-scoped rules + multi-language benchmark corpora
Rebuild the quick-scan detection engine and ground-truth benchmarks to hit >=90% recall / <10% false positives for C/C++, Python, JavaScript, and Java. Engine (findings.py, preprocess.py): - Scope each PatternRule by language (fixes cross-language false positives such as the Python eval/exec rule firing on JS .exec()). - Add JavaScript/TypeScript rules (command exec, eval/Function, reflected XSS, SQLi, path traversal, SSRF, weak hash, deserialization); execFile/spawn-argv excluded as safe alternatives. - Add Java rules (Runtime exec, SQL string concatenation, weak hash, readObject) and a Groovy server-side template rule (yieldUnescaped of a bound variable). - Broaden C/C++ and Python coverage; precision guards for format strings and parameterised vs concatenated SQL; #-comment skipping; CWE in rationale. - Register .java/.groovy/.tpl source extensions. Benchmarks: - Add realistic corpora modeled on Damn Vulnerable C/C++, PyGoat/DVPWA, and NodeGoat, plus the real kiview/damn-vulnerable-spring-boot-app vendored verbatim (MIT, attribution in LICENSE/NOTICE.md). Each manifest carries complete, independently-labeled ground truth including genuinely hard-to-detect vulns (integer overflow, second-order SQLi, prototype pollution) so recall is measured honestly, plus safe-API files for precision. - Fix pre-existing smoke-manifest ground-truth errors (Python/JS line offsets, missing C/C++ strcpy entry, Java SQLi line). benchmark.py: - Fix sink matching to normalise the expected sink consistently so sinks with underscores (e.g. render_template_string) can match. Tests: - Add tests/test_detection_benchmarks.py enforcing the >=90% recall / <10% FP goal per language and overall. Measured: C/C++ 93.3%, Python 93.3%, JavaScript 92.3%, Java 100%; overall 93.6% recall, 0.0% false-positive rate. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
be65aed96e
commit
cf9f945992
@@ -0,0 +1,38 @@
|
||||
name = "c-cpp-smoke"
|
||||
language = "c_cpp"
|
||||
frameworks = ["native"]
|
||||
setup = []
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/c-cpp-smoke"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "unsafe memory copy"
|
||||
path = "src/vuln.cpp"
|
||||
line = 4
|
||||
rule_id = "c-unsafe-memory-copy"
|
||||
sink = "memcpy"
|
||||
evidence = "memcpy copies attacker-controlled input into a fixed buffer"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "buffer overflow"
|
||||
path = "src/vuln.cpp"
|
||||
line = 8
|
||||
rule_id = "c-unsafe-copy"
|
||||
sink = "strcpy"
|
||||
evidence = "strcpy copies attacker-controlled input into a fixed buffer"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "src/vuln.cpp"
|
||||
line = 9
|
||||
rule_id = "c-shell-exec"
|
||||
sink = "system"
|
||||
evidence = "system executes attacker-controlled shell command"
|
||||
@@ -0,0 +1,118 @@
|
||||
name = "cpp-damn-vulnerable"
|
||||
language = "c_cpp"
|
||||
frameworks = ["native"]
|
||||
setup = []
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/cpp-damn-vulnerable"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "buffer overflow"
|
||||
path = "src/buffer.cpp"
|
||||
line = 8
|
||||
rule_id = "c-unsafe-copy"
|
||||
sink = "strcpy"
|
||||
evidence = "strcpy copies attacker input into a fixed buffer"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "buffer overflow"
|
||||
path = "src/buffer.cpp"
|
||||
line = 13
|
||||
rule_id = "c-unsafe-copy"
|
||||
sink = "strcat"
|
||||
evidence = "strcat appends attacker input without bounds"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "buffer overflow"
|
||||
path = "src/buffer.cpp"
|
||||
line = 18
|
||||
rule_id = "c-unsafe-copy"
|
||||
sink = "gets"
|
||||
evidence = "gets reads input without any bounds checking"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "buffer overflow"
|
||||
path = "src/buffer.cpp"
|
||||
line = 23
|
||||
rule_id = "c-unsafe-copy"
|
||||
sink = "sprintf"
|
||||
evidence = "sprintf formats attacker input into a fixed buffer"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "buffer overflow"
|
||||
path = "src/buffer.cpp"
|
||||
line = 27
|
||||
rule_id = "c-unsafe-memory-copy"
|
||||
sink = "memcpy"
|
||||
evidence = "memcpy copies an attacker-controlled length"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "buffer overflow"
|
||||
path = "src/buffer.cpp"
|
||||
line = 32
|
||||
rule_id = "c-unsafe-scanf"
|
||||
sink = "scanf"
|
||||
evidence = "scanf reads unbounded input into a fixed buffer"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-190"
|
||||
class = "integer overflow"
|
||||
path = "src/buffer.cpp"
|
||||
line = 37
|
||||
sink = "malloc"
|
||||
evidence = "malloc size len multiplied by count can overflow"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "buffer overflow"
|
||||
path = "src/command.cpp"
|
||||
line = 8
|
||||
rule_id = "c-unsafe-copy"
|
||||
sink = "strcpy"
|
||||
evidence = "strcpy copies the command argument into a fixed buffer"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "src/command.cpp"
|
||||
line = 9
|
||||
rule_id = "c-shell-exec"
|
||||
sink = "system"
|
||||
evidence = "system executes an attacker-controlled command"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "src/command.cpp"
|
||||
line = 13
|
||||
rule_id = "c-shell-exec"
|
||||
sink = "popen"
|
||||
evidence = "popen executes an attacker-controlled command"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "src/command.cpp"
|
||||
line = 18
|
||||
rule_id = "c-exec-family"
|
||||
sink = "execlp"
|
||||
evidence = "execlp executes an attacker-controlled argument"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-134"
|
||||
class = "format string"
|
||||
path = "src/command.cpp"
|
||||
line = 22
|
||||
rule_id = "c-unsafe-format"
|
||||
sink = "printf"
|
||||
evidence = "printf uses an attacker-controlled format string"
|
||||
@@ -0,0 +1,32 @@
|
||||
name = "java-spring-boot-vulnerable"
|
||||
language = "java_web"
|
||||
frameworks = ["spring-boot", "groovy-templates"]
|
||||
setup = ["./gradlew bootRun"]
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
# Source vendored verbatim from kiview/damn-vulnerable-spring-boot-app (MIT).
|
||||
# The app documents two vulnerabilities: SQL injection in the book detail view
|
||||
# and stored XSS in the comment view rendered by an unescaped Groovy template.
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/java-spring-boot-vulnerable"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "src/main/java/com/groovycoder/dvsba/books/BookController.java"
|
||||
line = 35
|
||||
rule_id = "java-sql-injection"
|
||||
sink = "select"
|
||||
evidence = "select query concatenates the request id parameter into SQL"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-79"
|
||||
class = "stored cross-site scripting"
|
||||
path = "src/main/resources/templates/views/comment/comment.tpl"
|
||||
line = 20
|
||||
rule_id = "groovy-xss-unescaped"
|
||||
sink = "yieldUnescaped"
|
||||
evidence = "yieldUnescaped renders the user comment without escaping"
|
||||
@@ -0,0 +1,29 @@
|
||||
name = "java-web-smoke"
|
||||
language = "java_web"
|
||||
frameworks = ["servlet"]
|
||||
setup = ["mvn test"]
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/java-web-smoke"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "src/main/java/example/VulnerableServlet.java"
|
||||
line = 8
|
||||
rule_id = "java-command-exec"
|
||||
sink = "exec"
|
||||
evidence = "exec runs the request parameter as a system command"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "src/main/java/example/VulnerableServlet.java"
|
||||
line = 12
|
||||
rule_id = "java-sql-injection"
|
||||
sink = "select"
|
||||
evidence = "select query concatenates request parameter input into SQL"
|
||||
@@ -0,0 +1,27 @@
|
||||
name = "javascript-node-web-smoke"
|
||||
language = "javascript_node_web"
|
||||
frameworks = ["node", "express"]
|
||||
setup = ["npm install"]
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/javascript-node-web-smoke"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "src/app.js"
|
||||
line = 6
|
||||
sink = "exec"
|
||||
evidence = "child_process.exec receives request query input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-79"
|
||||
class = "reflected cross-site scripting"
|
||||
path = "src/app.js"
|
||||
line = 10
|
||||
sink = "send"
|
||||
evidence = "response sends unsanitized request query input into HTML"
|
||||
@@ -0,0 +1,109 @@
|
||||
name = "javascript-vulnerable"
|
||||
language = "javascript_node_web"
|
||||
frameworks = ["node", "express"]
|
||||
setup = ["npm install"]
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/javascript-vulnerable"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "src/app.js"
|
||||
line = 10
|
||||
rule_id = "js-command-exec"
|
||||
sink = "exec"
|
||||
evidence = "child_process exec runs an attacker-controlled command"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "src/app.js"
|
||||
line = 14
|
||||
rule_id = "js-command-exec"
|
||||
sink = "execSync"
|
||||
evidence = "execSync runs an attacker-controlled command"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-95"
|
||||
class = "code injection"
|
||||
path = "src/app.js"
|
||||
line = 18
|
||||
rule_id = "js-eval"
|
||||
sink = "eval"
|
||||
evidence = "eval executes attacker-controlled input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-95"
|
||||
class = "code injection"
|
||||
path = "src/app.js"
|
||||
line = 23
|
||||
rule_id = "js-function-constructor"
|
||||
sink = "Function"
|
||||
evidence = "Function constructor builds code from attacker input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-79"
|
||||
class = "reflected cross-site scripting"
|
||||
path = "src/app.js"
|
||||
line = 28
|
||||
rule_id = "js-xss-response"
|
||||
sink = "send"
|
||||
evidence = "response send reflects request query input into HTML"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "src/app.js"
|
||||
line = 32
|
||||
rule_id = "js-sql-injection"
|
||||
sink = "query"
|
||||
evidence = "db query concatenates request input into SQL"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "src/app.js"
|
||||
line = 36
|
||||
rule_id = "js-sql-injection"
|
||||
sink = "query"
|
||||
evidence = "db query interpolates request input via a template literal"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-22"
|
||||
class = "path traversal"
|
||||
path = "src/app.js"
|
||||
line = 40
|
||||
rule_id = "js-path-traversal"
|
||||
sink = "readFileSync"
|
||||
evidence = "fs readFileSync reads an attacker-controlled path"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-918"
|
||||
class = "server-side request forgery"
|
||||
path = "src/app.js"
|
||||
line = 45
|
||||
rule_id = "js-ssrf"
|
||||
sink = "axios"
|
||||
evidence = "axios get fetches an attacker-controlled url"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-327"
|
||||
class = "weak cryptography"
|
||||
path = "src/app.js"
|
||||
line = 49
|
||||
rule_id = "js-weak-hash"
|
||||
sink = "md5"
|
||||
evidence = "createHash md5 is a weak hash"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-1321"
|
||||
class = "prototype pollution"
|
||||
path = "src/app.js"
|
||||
line = 55
|
||||
sink = "target"
|
||||
evidence = "user-controlled keys are copied into an object enabling prototype pollution"
|
||||
@@ -0,0 +1,127 @@
|
||||
name = "python-vulnerable"
|
||||
language = "python_web"
|
||||
frameworks = ["flask"]
|
||||
setup = ["pip install flask pyyaml requests"]
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/python-vulnerable"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "app.py"
|
||||
line = 17
|
||||
rule_id = "python-os-command"
|
||||
sink = "os.system"
|
||||
evidence = "os.system runs an attacker-controlled command"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "app.py"
|
||||
line = 23
|
||||
rule_id = "python-shell-true"
|
||||
sink = "subprocess"
|
||||
evidence = "subprocess runs an attacker-controlled command with shell=True"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-95"
|
||||
class = "code injection"
|
||||
path = "app.py"
|
||||
line = 28
|
||||
rule_id = "python-unsafe-eval"
|
||||
sink = "eval"
|
||||
evidence = "eval executes attacker-controlled input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-502"
|
||||
class = "insecure deserialization"
|
||||
path = "app.py"
|
||||
line = 33
|
||||
rule_id = "python-unsafe-deserialization"
|
||||
sink = "pickle.loads"
|
||||
evidence = "pickle.loads deserializes the request body"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-502"
|
||||
class = "insecure deserialization"
|
||||
path = "app.py"
|
||||
line = 38
|
||||
rule_id = "python-unsafe-deserialization"
|
||||
sink = "yaml.load"
|
||||
evidence = "yaml.load deserializes untrusted input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "app.py"
|
||||
line = 44
|
||||
rule_id = "python-sql-injection"
|
||||
sink = "execute"
|
||||
evidence = "execute concatenates request input into the SQL query"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "app.py"
|
||||
line = 50
|
||||
rule_id = "python-sql-injection"
|
||||
sink = "execute"
|
||||
evidence = "execute uses percent formatted request input in SQL"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-22"
|
||||
class = "path traversal"
|
||||
path = "app.py"
|
||||
line = 55
|
||||
rule_id = "python-path-traversal"
|
||||
sink = "open"
|
||||
evidence = "open reads an attacker-controlled file path"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-918"
|
||||
class = "server-side request forgery"
|
||||
path = "app.py"
|
||||
line = 60
|
||||
rule_id = "python-ssrf"
|
||||
sink = "requests.get"
|
||||
evidence = "requests.get fetches an attacker-controlled url"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-327"
|
||||
class = "weak cryptography"
|
||||
path = "app.py"
|
||||
line = 65
|
||||
rule_id = "python-weak-hash"
|
||||
sink = "md5"
|
||||
evidence = "hashlib.md5 is a weak hash"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-94"
|
||||
class = "server-side template injection"
|
||||
path = "app.py"
|
||||
line = 70
|
||||
rule_id = "python-ssti"
|
||||
sink = "render_template_string"
|
||||
evidence = "render_template_string evaluates attacker-controlled input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "app.py"
|
||||
line = 77
|
||||
sink = "execute"
|
||||
evidence = "execute runs a SQL query string built from request input on a prior line"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-489"
|
||||
class = "insecure default"
|
||||
path = "app.py"
|
||||
line = 81
|
||||
rule_id = "python-flask-debug"
|
||||
sink = "debug"
|
||||
evidence = "flask debug mode is enabled in production"
|
||||
@@ -0,0 +1,29 @@
|
||||
name = "python-web-smoke"
|
||||
language = "python_web"
|
||||
frameworks = ["flask"]
|
||||
setup = ["pip install flask"]
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/python-web-smoke"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "app.py"
|
||||
line = 9
|
||||
rule_id = "python-shell-true"
|
||||
sink = "subprocess"
|
||||
evidence = "subprocess executes request input with shell=True"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-94"
|
||||
class = "code injection"
|
||||
path = "app.py"
|
||||
line = 13
|
||||
rule_id = "python-unsafe-eval"
|
||||
sink = "eval"
|
||||
evidence = "eval executes request input"
|
||||
Reference in New Issue
Block a user