marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-17 12:09:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pair-corpus-honesty): function-scoped honest scorer with mechanisms and loss counters

Browse Source 
Tasks 1.2, 2.1-2.5 and 6.2 (pair-side filters).

- pairs.py: coverage with a source counts as detection and as leak;
  function-scoped matching by line-range containment from the vulnerable
  side's parse (unresolvable labels are counted loss, never silent
  misses); exact or alias sink matching; weak CWE-only labels never match;
  side-by-side inventory/overlay/hunter scorers; per_profile,
  per_mechanism, achievable, known_limit and loss payload; loader rejects
  unknown mechanism/provenance/split and reserved or missing function
  labels on harvested slices; candidate rulesets thread through for the
  improve loop.
- overlay.py: OverlayRecord.function (additive) filled from the shared
  naming helper in one parse; benchmark.py: ExpectedFinding.mechanism.
- cli pairs: --slice over every slice, --profile, --split, --hunter,
  --hunter-model.
- catalogs migrated to the new schema (provenance, split, function,
  mechanism, known_limit for owasp-java-hash); baseline recorded in the
  spec notes.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-05 10:45:43 +02:00
co-authored by Claude Fable 5.1
parent 8ab4207618
commit c0b9e41000
9 changed files with 1915 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/pairs/catalog.toml
+18
View File
@@ -55,6 +55,8 @@ fixed = "github/sven-cwe-078-os-system/fixed.py"
relpath = "vuedj/configtitania/views.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
[[pair.expected]]
cwe = "CWE-78"
@@ -63,6 +65,7 @@ path = "vuedj/configtitania/views.py"
rule_id = "python-os-command"
sink = "os.system"
evidence = "os.system concatenates attacker-controlled username into a shell command"
mechanism = "source_reaches_sink"
[[pair]]
name = "sven-cwe-078-shell-true"
@@ -78,6 +81,8 @@ fixed = "github/sven-cwe-078-shell-true/fixed.py"
relpath = "mobly/controllers/android_device_lib/adb.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
[[pair.expected]]
cwe = "CWE-78"
@@ -86,6 +91,7 @@ path = "mobly/controllers/android_device_lib/adb.py"
rule_id = "python-shell-true"
sink = "subprocess"
evidence = "subprocess.Popen runs the command with shell=True"
mechanism = "source_reaches_sink"
[[pair]]
name = "sven-cwe-089-execute-percent"
@@ -101,6 +107,8 @@ fixed = "github/sven-cwe-089-execute-percent/fixed.py"
relpath = "cgi/common.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
[[pair.expected]]
cwe = "CWE-89"
@@ -109,6 +117,7 @@ path = "cgi/common.py"
rule_id = "python-sql-injection"
sink = "execute"
evidence = "execute percent-formats attacker-controlled input into SQL"
mechanism = "source_reaches_sink"
[[pair]]
name = "sven-cwe-089-query-concat"
@@ -124,6 +133,8 @@ fixed = "github/sven-cwe-089-query-concat/fixed.py"
relpath = "modules/query_lastfm.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
[[pair.expected]]
cwe = "CWE-89"
@@ -131,6 +142,7 @@ class = "sql injection"
path = "modules/query_lastfm.py"
sink = "SELECT"
evidence = "SQL query concatenates attacker-controlled tag_num"
mechanism = "source_reaches_sink"
[[pair]]
name = "hutool-cve-2018-17297"
@@ -149,6 +161,8 @@ fixed = "github/hutool-cve-2018-17297/fixed/ZipUtil.java"
relpath = "cn/hutool/core/util/ZipUtil.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "train"
[[pair.expected]]
cwe = "CWE-22"
@@ -156,6 +170,7 @@ class = "path traversal"
path = "cn/hutool/core/util/ZipUtil.java"
sink = "ZipEntry"
evidence = "zip entry name is joined onto the extract directory without a zip-slip check"
mechanism = "path_join_user_input"
[[pair]]
name = "node-serialize-cve-2017-5941"
@@ -171,6 +186,8 @@ fixed = "github/node-serialize-cve-2017-5941/fixed.js"
relpath = "index.js"
min_recall = 1.0
fix_policy = "silent"
provenance = "human"
split = "holdout"
[[pair.expected]]
cwe = "CWE-95"
@@ -179,3 +196,4 @@ path = "index.js"
rule_id = "js-eval"
sink = "eval"
evidence = "eval executes attacker-controlled serialized input"
mechanism = "source_reaches_sink"
benchmarks/pairs/sast/catalog.toml
+34
View File
@@ -17,6 +17,8 @@ fixed = "owasp-python/BenchmarkTest00073.py"
relpath = "testcode/BenchmarkTest00074.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-94"
@@ -25,6 +27,7 @@ path = "testcode/BenchmarkTest00074.py"
rule_id = "python-unsafe-eval"
sink = "exec"
evidence = "exec executes attacker-controlled input"
mechanism = "container_taint"
[[pair]]
name = "owasp-python-deser"
@@ -39,6 +42,8 @@ fixed = "owasp-python/BenchmarkTest00079.py"
relpath = "testcode/BenchmarkTest00078.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-502"
@@ -47,6 +52,7 @@ path = "testcode/BenchmarkTest00078.py"
rule_id = "python-unsafe-deserialization"
sink = "pickle"
evidence = "pickle.loads deserializes untrusted input"
mechanism = "source_reaches_sink"
[[pair]]
name = "owasp-python-cmdi"
@@ -61,6 +67,8 @@ fixed = "owasp-python/BenchmarkTest00350.py"
relpath = "testcode/BenchmarkTest00165.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-78"
@@ -69,6 +77,7 @@ path = "testcode/BenchmarkTest00165.py"
rule_id = "python-shell-true"
sink = "subprocess"
evidence = "subprocess runs an attacker-controlled command"
mechanism = "container_taint"
[[pair]]
name = "owasp-python-sqli"
@@ -83,6 +92,8 @@ fixed = "owasp-python/BenchmarkTest00011.py"
relpath = "testcode/BenchmarkTest00099.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-89"
@@ -91,6 +102,7 @@ path = "testcode/BenchmarkTest00099.py"
rule_id = "python-sql-injection"
sink = "execute"
evidence = "execute runs a SQL string built from request input"
mechanism = "container_taint"
[[pair]]
name = "owasp-python-hash"
@@ -105,6 +117,8 @@ fixed = "owasp-python/BenchmarkTest00055.py"
relpath = "testcode/BenchmarkTest00054.py"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-327"
@@ -113,6 +127,7 @@ path = "testcode/BenchmarkTest00054.py"
rule_id = "python-weak-hash"
sink = "md5"
evidence = "md5 is a weak hash"
mechanism = "config_sink_weak_literal"
# --- OWASP Benchmark Java v1.2 ---
@@ -129,6 +144,8 @@ fixed = "owasp-java/BenchmarkTest00090.java"
relpath = "org/owasp/benchmark/testcode/BenchmarkTest00007.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-78"
@@ -137,6 +154,7 @@ path = "org/owasp/benchmark/testcode/BenchmarkTest00007.java"
rule_id = "java-command-exec"
sink = "exec"
evidence = "Runtime.exec runs an attacker-controlled command"
mechanism = "source_reaches_sink"
[[pair]]
name = "owasp-java-sqli"
@@ -151,6 +169,8 @@ fixed = "owasp-java/BenchmarkTest00052.java"
relpath = "org/owasp/benchmark/testcode/BenchmarkTest00008.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-89"
@@ -159,6 +179,7 @@ path = "org/owasp/benchmark/testcode/BenchmarkTest00008.java"
rule_id = "java-sql-injection"
sink = "prepareCall"
evidence = "SQL is concatenated with request input"
mechanism = "source_reaches_sink"
[[pair]]
name = "owasp-java-hash"
@@ -173,6 +194,9 @@ fixed = "owasp-java/BenchmarkTest00009.java"
relpath = "org/owasp/benchmark/testcode/BenchmarkTest00003.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
known_limit = "cross_artifact: OWASP labels this true via hashAlg1=MD5 in benchmark.properties, which is not in the vendored file"
[[pair.expected]]
cwe = "CWE-327"
@@ -181,6 +205,7 @@ path = "org/owasp/benchmark/testcode/BenchmarkTest00003.java"
rule_id = "java-weak-hash"
sink = "MessageDigest"
evidence = "MessageDigest uses a weak hash algorithm"
mechanism = "cross_artifact"
# --- Juliet C/C++ 1.3 (CC0) ---
@@ -197,6 +222,8 @@ fixed = "juliet-c/cwe134-printf-fixed.c"
relpath = "CWE134_char_console_printf_01.c"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-134"
@@ -205,6 +232,7 @@ path = "CWE134_char_console_printf_01.c"
rule_id = "c-unsafe-format"
sink = "printf"
evidence = "printf uses attacker-controlled format string"
mechanism = "source_reaches_sink"
[[pair]]
name = "juliet-c-cwe121-strcpy"
@@ -219,6 +247,8 @@ fixed = "juliet-c/cwe121-strcpy-fixed.c"
relpath = "CWE121_char_declare_cpy_01.c"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "holdout"
[[pair.expected]]
cwe = "CWE-120"
@@ -227,6 +257,7 @@ path = "CWE121_char_declare_cpy_01.c"
rule_id = "c-unsafe-copy"
sink = "strcpy"
evidence = "strcpy copies into a buffer without a null terminator"
mechanism = "fixed_buffer_capacity"
# --- Juliet Java 1.3 (CC0) ---
@@ -243,6 +274,8 @@ fixed = "juliet-java/cwe78-exec-fixed.java"
relpath = "CWE78_OS_Command_Injection__console_readLine_01.java"
min_recall = 1.0
fix_policy = "silent"
provenance = "synthetic"
split = "train"
[[pair.expected]]
cwe = "CWE-78"
@@ -251,3 +284,4 @@ path = "CWE78_OS_Command_Injection__console_readLine_01.java"
rule_id = "java-command-exec"
sink = "exec"
evidence = "Runtime.exec concatenates attacker-controlled input into a command"
mechanism = "source_reaches_sink"
benchmarks/pairs/vfc/catalog.toml
+706 -2
View File
File diff suppressed because it is too large Load Diff