From c0b9e41000f9e1fc63744e09cd169c6e9e3e3d46 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sat, 5 Sep 2026 09:27:40 +0200 Subject: [PATCH] feat(pair-corpus-honesty): function-scoped honest scorer with mechanisms and loss counters Tasks 1.2, 2.1-2.5 and 6.2 (pair-side filters). - pairs.py: coverage with a source counts as detection and as leak; function-scoped matching by line-range containment from the vulnerable side's parse (unresolvable labels are counted loss, never silent misses); exact or alias sink matching; weak CWE-only labels never match; side-by-side inventory/overlay/hunter scorers; per_profile, per_mechanism, achievable, known_limit and loss payload; loader rejects unknown mechanism/provenance/split and reserved or missing function labels on harvested slices; candidate rulesets thread through for the improve loop. - overlay.py: OverlayRecord.function (additive) filled from the shared naming helper in one parse; benchmark.py: ExpectedFinding.mechanism. - cli pairs: --slice over every slice, --profile, --split, --hunter, --hunter-model. - catalogs migrated to the new schema (provenance, split, function, mechanism, known_limit for owasp-java-hash); baseline recorded in the spec notes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .env.example | 7 + benchmarks/pairs/catalog.toml | 18 + benchmarks/pairs/sast/catalog.toml | 34 ++ benchmarks/pairs/vfc/catalog.toml | 708 +++++++++++++++++++++++++- src/openultrasast/benchmark.py | 2 + src/openultrasast/cli.py | 53 +- src/openultrasast/pairs.py | 627 +++++++++++++++++++---- src/openultrasast/semantic/overlay.py | 39 +- tests/test_pair_corpus.py | 554 ++++++++++++++++++++ 9 files changed, 1915 insertions(+), 127 deletions(-) diff --git a/.env.example b/.env.example index e967ae8..adc346b 100644 --- a/.env.example +++ b/.env.example @@ -3,3 +3,10 @@ OPENROUTER_API_KEY= # Prove-budget and `ousast index` embeddings via OpenRouter /embeddings. OPENROUTER_EMBEDDING_MODEL=openai/text-embedding-3-small + +# Any OpenAI-compatible chat endpoint works for the hunter: set the key above to that +# provider's key and override the base URL. Example for DeepSeek: +# OPENROUTER_API_KEY=sk-deepseek-... +# OPENROUTER_BASE_URL=https://api.deepseek.com/v1 +# Then set [models].hunter in openultrasast.toml (or --hunter-model) to that provider's model id. +OPENROUTER_BASE_URL= diff --git a/benchmarks/pairs/catalog.toml b/benchmarks/pairs/catalog.toml index 98a9f83..1c91f18 100644 --- a/benchmarks/pairs/catalog.toml +++ b/benchmarks/pairs/catalog.toml @@ -55,6 +55,8 @@ fixed = "github/sven-cwe-078-os-system/fixed.py" relpath = "vuedj/configtitania/views.py" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-78" @@ -63,6 +65,7 @@ path = "vuedj/configtitania/views.py" rule_id = "python-os-command" sink = "os.system" evidence = "os.system concatenates attacker-controlled username into a shell command" +mechanism = "source_reaches_sink" [[pair]] name = "sven-cwe-078-shell-true" @@ -78,6 +81,8 @@ fixed = "github/sven-cwe-078-shell-true/fixed.py" relpath = "mobly/controllers/android_device_lib/adb.py" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-78" @@ -86,6 +91,7 @@ path = "mobly/controllers/android_device_lib/adb.py" rule_id = "python-shell-true" sink = "subprocess" evidence = "subprocess.Popen runs the command with shell=True" +mechanism = "source_reaches_sink" [[pair]] name = "sven-cwe-089-execute-percent" @@ -101,6 +107,8 @@ fixed = "github/sven-cwe-089-execute-percent/fixed.py" relpath = "cgi/common.py" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-89" @@ -109,6 +117,7 @@ path = "cgi/common.py" rule_id = "python-sql-injection" sink = "execute" evidence = "execute percent-formats attacker-controlled input into SQL" +mechanism = "source_reaches_sink" [[pair]] name = "sven-cwe-089-query-concat" @@ -124,6 +133,8 @@ fixed = "github/sven-cwe-089-query-concat/fixed.py" relpath = "modules/query_lastfm.py" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-89" @@ -131,6 +142,7 @@ class = "sql injection" path = "modules/query_lastfm.py" sink = "SELECT" evidence = "SQL query concatenates attacker-controlled tag_num" +mechanism = "source_reaches_sink" [[pair]] name = "hutool-cve-2018-17297" @@ -149,6 +161,8 @@ fixed = "github/hutool-cve-2018-17297/fixed/ZipUtil.java" relpath = "cn/hutool/core/util/ZipUtil.java" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-22" @@ -156,6 +170,7 @@ class = "path traversal" path = "cn/hutool/core/util/ZipUtil.java" sink = "ZipEntry" evidence = "zip entry name is joined onto the extract directory without a zip-slip check" +mechanism = "path_join_user_input" [[pair]] name = "node-serialize-cve-2017-5941" @@ -171,6 +186,8 @@ fixed = "github/node-serialize-cve-2017-5941/fixed.js" relpath = "index.js" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-95" @@ -179,3 +196,4 @@ path = "index.js" rule_id = "js-eval" sink = "eval" evidence = "eval executes attacker-controlled serialized input" +mechanism = "source_reaches_sink" diff --git a/benchmarks/pairs/sast/catalog.toml b/benchmarks/pairs/sast/catalog.toml index a923f63..5d314f7 100644 --- a/benchmarks/pairs/sast/catalog.toml +++ b/benchmarks/pairs/sast/catalog.toml @@ -17,6 +17,8 @@ fixed = "owasp-python/BenchmarkTest00073.py" relpath = "testcode/BenchmarkTest00074.py" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "train" [[pair.expected]] cwe = "CWE-94" @@ -25,6 +27,7 @@ path = "testcode/BenchmarkTest00074.py" rule_id = "python-unsafe-eval" sink = "exec" evidence = "exec executes attacker-controlled input" +mechanism = "container_taint" [[pair]] name = "owasp-python-deser" @@ -39,6 +42,8 @@ fixed = "owasp-python/BenchmarkTest00079.py" relpath = "testcode/BenchmarkTest00078.py" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "holdout" [[pair.expected]] cwe = "CWE-502" @@ -47,6 +52,7 @@ path = "testcode/BenchmarkTest00078.py" rule_id = "python-unsafe-deserialization" sink = "pickle" evidence = "pickle.loads deserializes untrusted input" +mechanism = "source_reaches_sink" [[pair]] name = "owasp-python-cmdi" @@ -61,6 +67,8 @@ fixed = "owasp-python/BenchmarkTest00350.py" relpath = "testcode/BenchmarkTest00165.py" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "train" [[pair.expected]] cwe = "CWE-78" @@ -69,6 +77,7 @@ path = "testcode/BenchmarkTest00165.py" rule_id = "python-shell-true" sink = "subprocess" evidence = "subprocess runs an attacker-controlled command" +mechanism = "container_taint" [[pair]] name = "owasp-python-sqli" @@ -83,6 +92,8 @@ fixed = "owasp-python/BenchmarkTest00011.py" relpath = "testcode/BenchmarkTest00099.py" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "holdout" [[pair.expected]] cwe = "CWE-89" @@ -91,6 +102,7 @@ path = "testcode/BenchmarkTest00099.py" rule_id = "python-sql-injection" sink = "execute" evidence = "execute runs a SQL string built from request input" +mechanism = "container_taint" [[pair]] name = "owasp-python-hash" @@ -105,6 +117,8 @@ fixed = "owasp-python/BenchmarkTest00055.py" relpath = "testcode/BenchmarkTest00054.py" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "holdout" [[pair.expected]] cwe = "CWE-327" @@ -113,6 +127,7 @@ path = "testcode/BenchmarkTest00054.py" rule_id = "python-weak-hash" sink = "md5" evidence = "md5 is a weak hash" +mechanism = "config_sink_weak_literal" # --- OWASP Benchmark Java v1.2 --- @@ -129,6 +144,8 @@ fixed = "owasp-java/BenchmarkTest00090.java" relpath = "org/owasp/benchmark/testcode/BenchmarkTest00007.java" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "train" [[pair.expected]] cwe = "CWE-78" @@ -137,6 +154,7 @@ path = "org/owasp/benchmark/testcode/BenchmarkTest00007.java" rule_id = "java-command-exec" sink = "exec" evidence = "Runtime.exec runs an attacker-controlled command" +mechanism = "source_reaches_sink" [[pair]] name = "owasp-java-sqli" @@ -151,6 +169,8 @@ fixed = "owasp-java/BenchmarkTest00052.java" relpath = "org/owasp/benchmark/testcode/BenchmarkTest00008.java" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "holdout" [[pair.expected]] cwe = "CWE-89" @@ -159,6 +179,7 @@ path = "org/owasp/benchmark/testcode/BenchmarkTest00008.java" rule_id = "java-sql-injection" sink = "prepareCall" evidence = "SQL is concatenated with request input" +mechanism = "source_reaches_sink" [[pair]] name = "owasp-java-hash" @@ -173,6 +194,9 @@ fixed = "owasp-java/BenchmarkTest00009.java" relpath = "org/owasp/benchmark/testcode/BenchmarkTest00003.java" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "train" +known_limit = "cross_artifact: OWASP labels this true via hashAlg1=MD5 in benchmark.properties, which is not in the vendored file" [[pair.expected]] cwe = "CWE-327" @@ -181,6 +205,7 @@ path = "org/owasp/benchmark/testcode/BenchmarkTest00003.java" rule_id = "java-weak-hash" sink = "MessageDigest" evidence = "MessageDigest uses a weak hash algorithm" +mechanism = "cross_artifact" # --- Juliet C/C++ 1.3 (CC0) --- @@ -197,6 +222,8 @@ fixed = "juliet-c/cwe134-printf-fixed.c" relpath = "CWE134_char_console_printf_01.c" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "train" [[pair.expected]] cwe = "CWE-134" @@ -205,6 +232,7 @@ path = "CWE134_char_console_printf_01.c" rule_id = "c-unsafe-format" sink = "printf" evidence = "printf uses attacker-controlled format string" +mechanism = "source_reaches_sink" [[pair]] name = "juliet-c-cwe121-strcpy" @@ -219,6 +247,8 @@ fixed = "juliet-c/cwe121-strcpy-fixed.c" relpath = "CWE121_char_declare_cpy_01.c" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "holdout" [[pair.expected]] cwe = "CWE-120" @@ -227,6 +257,7 @@ path = "CWE121_char_declare_cpy_01.c" rule_id = "c-unsafe-copy" sink = "strcpy" evidence = "strcpy copies into a buffer without a null terminator" +mechanism = "fixed_buffer_capacity" # --- Juliet Java 1.3 (CC0) --- @@ -243,6 +274,8 @@ fixed = "juliet-java/cwe78-exec-fixed.java" relpath = "CWE78_OS_Command_Injection__console_readLine_01.java" min_recall = 1.0 fix_policy = "silent" +provenance = "synthetic" +split = "train" [[pair.expected]] cwe = "CWE-78" @@ -251,3 +284,4 @@ path = "CWE78_OS_Command_Injection__console_readLine_01.java" rule_id = "java-command-exec" sink = "exec" evidence = "Runtime.exec concatenates attacker-controlled input into a command" +mechanism = "source_reaches_sink" diff --git a/benchmarks/pairs/vfc/catalog.toml b/benchmarks/pairs/vfc/catalog.toml index bf2e782..95f961f 100644 --- a/benchmarks/pairs/vfc/catalog.toml +++ b/benchmarks/pairs/vfc/catalog.toml @@ -1,7 +1,7 @@ -# Real-world VFC slice. Reviewed isolated functions. +# vfc pair slice. Reviewed isolated functions with provenance. # Honesty dashboard — not the local 100% CI gate. # Do not add these names to LANGUAGE_MANIFESTS. -# Regenerated by generate_catalog.py from recipes.toml. +# Regenerated by benchmarks/pairs/catalog_gen.py from recipes.toml. [[pair]] name = "openssl-cve-2014-0160" @@ -20,11 +20,15 @@ fixed = "openssl-cve-2014-0160/fixed.c" relpath = "ssl/t1_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-126" class = "buffer over-read" path = "ssl/t1_lib.c" +function = "tls1_process_heartbeat" +mechanism = "unchecked_length_copy" rule_id = "c-unsafe-memory-copy" sink = "memcpy" evidence = "memcpy copies heartbeat payload using an attacker-controlled length" @@ -46,11 +50,15 @@ fixed = "firefox-cve-2020-15667/fixed.c" relpath = "modules/libmar/src/mar_read.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "modules/libmar/src/mar_read.c" +function = "mar_insert_item" +mechanism = "unchecked_length_copy" rule_id = "c-unsafe-memory-copy" sink = "memcpy" evidence = "memcpy copies a MAR index name using a signed namelen that can wrap" @@ -72,11 +80,15 @@ fixed = "chromium-cve-2019-5786/fixed.cc" relpath = "third_party/blink/renderer/core/fileapi/file_reader_loader.cc" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-416" class = "use after free" path = "third_party/blink/renderer/core/fileapi/file_reader_loader.cc" +function = "ArrayBufferResult" +mechanism = "uaf" rule_id = "c-unsafe-memory-copy" sink = "ToArrayBuffer" evidence = "ArrayBufferResult can return two references to the same underlying ArrayBuffer" @@ -98,11 +110,15 @@ fixed = "openssl-cve-2020-1967/fixed.c" relpath = "ssl/t1_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "ssl/t1_lib.c" +function = "tls1_check_sig_alg" +mechanism = "null_deref_unguarded" sink = "tls1_check_sig_alg" evidence = "SSL_check_chain / tls1_check_sig_alg can dereference NULL in TLS 1.3" @@ -123,11 +139,15 @@ fixed = "openssl-cve-2020-1971/fixed.c" relpath = "crypto/x509/v3_genn.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/x509/v3_genn.c" +function = "GENERAL_NAME_cmp" +mechanism = "null_deref_unguarded" sink = "GENERAL_NAME_cmp" evidence = "GENERAL_NAME_cmp mishandles EdiPartyName comparison" @@ -148,11 +168,15 @@ fixed = "openssl-cve-2021-23840/fixed.c" relpath = "crypto/evp/evp_enc.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-190" class = "integer overflow" path = "crypto/evp/evp_enc.c" +function = "evp_EncryptDecryptUpdate" +mechanism = "signed_size_operand" sink = "EVP_CipherUpdate" evidence = "EVP_CipherUpdate output length can overflow" @@ -173,11 +197,15 @@ fixed = "openssl-cve-2021-3711/fixed.c" relpath = "crypto/sm2/sm2_crypt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-120" class = "buffer overflow" path = "crypto/sm2/sm2_crypt.c" +function = "sm2_plaintext_size" +mechanism = "unchecked_length_copy" sink = "sm2_decrypt" evidence = "SM2 plaintext size is underestimated so the second EVP_PKEY_decrypt overflows" @@ -198,11 +226,15 @@ fixed = "openssl-cve-2021-3712/fixed.c" relpath = "crypto/x509/v3_utl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-125" class = "buffer over-read" path = "crypto/x509/v3_utl.c" +function = "append_ia5" +mechanism = "unchecked_length_copy" sink = "append_ia5" evidence = "append_ia5 assumes NUL-terminated ASN.1 strings" @@ -223,11 +255,15 @@ fixed = "openssl-cve-2022-0778/fixed.c" relpath = "crypto/bn/bn_sqrt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-835" class = "infinite loop" path = "crypto/bn/bn_sqrt.c" +function = "BN_mod_sqrt" +mechanism = "other" sink = "BN_mod_sqrt" evidence = "BN_mod_sqrt can loop forever on crafted moduli" @@ -248,11 +284,15 @@ fixed = "openssl-cve-2022-1292/fixed.in" relpath = "tools/c_rehash.in" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-78" class = "command injection" path = "tools/c_rehash.in" +function = "link_hash_cert" +mechanism = "source_reaches_sink" sink = "backtick" evidence = "c_rehash interpolates certificate filenames into a shell backtick command" @@ -273,11 +313,15 @@ fixed = "openssl-cve-2022-3602/fixed.c" relpath = "crypto/punycode.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-787" class = "buffer overflow" path = "crypto/punycode.c" +function = "ossl_punycode_decode" +mechanism = "unchecked_length_copy" sink = "ossl_punycode_decode" evidence = "punycode decoder can write past the output buffer" @@ -298,11 +342,15 @@ fixed = "openssl-cve-2022-3786/fixed.c" relpath = "crypto/punycode.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-787" class = "buffer overflow" path = "crypto/punycode.c" +function = "ossl_punycode_decode" +mechanism = "unchecked_length_copy" sink = "ossl_punycode_decode" evidence = "punycode decoder can overflow a stack buffer via a crafted email name" @@ -323,11 +371,15 @@ fixed = "openssl-cve-2023-0286/fixed.c" relpath = "crypto/x509/v3_genn.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-843" class = "type confusion" path = "crypto/x509/v3_genn.c" +function = "GENERAL_NAME_cmp" +mechanism = "type_confusion" sink = "GENERAL_NAME_cmp" evidence = "GENERAL_NAME_cmp type-confuses x400Address" @@ -348,11 +400,15 @@ fixed = "curl-cve-2023-38545/fixed.c" relpath = "lib/socks.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-122" class = "heap buffer overflow" path = "lib/socks.c" +function = "do_SOCKS5" +mechanism = "unchecked_length_copy" rule_id = "c-unsafe-memory-copy" sink = "memcpy" evidence = "SOCKS5 remote-resolve copies a hostname that can exceed the local buffer" @@ -374,11 +430,15 @@ fixed = "chromium-cve-2022-2162/fixed.cc" relpath = "content/browser/file_system_access/file_system_chooser.cc" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-20" class = "insufficient input validation" path = "content/browser/file_system_access/file_system_chooser.cc" +function = "IsShellIntegratedExtension" +mechanism = "validation_strength" sink = "IsShellIntegratedExtension" evidence = "File System Access suggested names did not sanitize Windows .url shell extensions" @@ -399,11 +459,15 @@ fixed = "openssl-cve-2025-69421/fixed.c" relpath = "crypto/pkcs12/p12_decr.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/pkcs12/p12_decr.c" +function = "PKCS12_item_decrypt_d2i_ex" +mechanism = "null_deref_unguarded" sink = "PKCS12_item_decrypt_d2i_ex" evidence = "PKCS12_item_decrypt_d2i_ex did not check a NULL oct argument" @@ -424,11 +488,15 @@ fixed = "openssl-cve-2025-66199/fixed.c" relpath = "ssl/statem/statem_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-770" class = "uncontrolled allocation" path = "ssl/statem/statem_lib.c" +function = "tls13_process_compressed_certificate" +mechanism = "unchecked_alloc_size" sink = "tls13_process_compressed_certificate" evidence = "TLS 1.3 compressed certificate length was trusted before allocation" @@ -449,11 +517,15 @@ fixed = "firefox-bug-2058173/fixed.c" relpath = "modules/libmar/src/mar_read.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-770" class = "uncontrolled allocation" path = "modules/libmar/src/mar_read.c" +function = "mar_read_index" +mechanism = "unchecked_alloc_size" sink = "malloc" evidence = "mar_read_index malloc'd an untrusted MAR index size with no upper bound" @@ -474,11 +546,15 @@ fixed = "openssl-cve-2026-18798/fixed.c" relpath = "ssl/quic/quic_port.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/quic/quic_port.c" +function = "port_default_packet_handler" +mechanism = "other" sink = "unknown" evidence = "Avoid double free of qrx in port_default_packet_handler()" @@ -499,11 +575,15 @@ fixed = "openssl-cve-2026-54874/fixed.c" relpath = "ssl/record/rec_layer_d1.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/record/rec_layer_d1.c" +function = "DTLS_RECORD_LAYER_clear" +mechanism = "other" sink = "unknown" evidence = "Avoid full read buffer allocation when buffering DTLS records" @@ -524,11 +604,15 @@ fixed = "openssl-cve-2026-54874-dtls1_process_record/fixed.c" relpath = "ssl/record/ssl3_record.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/record/ssl3_record.c" +function = "dtls1_process_record" +mechanism = "other" sink = "unknown" evidence = "Avoid full read buffer allocation when buffering DTLS records" @@ -549,11 +633,15 @@ fixed = "openssl-cve-2026-54874-dtls_rlayer_buffer_recor/fixed.c" relpath = "ssl/record/methods/dtls_meth.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/record/methods/dtls_meth.c" +function = "dtls_rlayer_buffer_record" +mechanism = "other" sink = "unknown" evidence = "Avoid full read buffer allocation when buffering DTLS next-epoch records" @@ -574,11 +662,15 @@ fixed = "openssl-cve-2026-63072/fixed.c" relpath = "crypto/cms/cms_kari.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "crypto/cms/cms_kari.c" +function = "cms_kek_cipher" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "Fix heap buffer overflow (8-byte OOB write) in AES-WRAP-PAD unwrap" @@ -599,11 +691,15 @@ fixed = "openssl-cve-2026-63072-cms_kek_cipher/fixed.c" relpath = "crypto/cms/cms_kemri.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "crypto/cms/cms_kemri.c" +function = "cms_kek_cipher" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "Fix heap buffer overflow (8-byte OOB write) in AES-WRAP-PAD unwrap" @@ -624,11 +720,15 @@ fixed = "openssl-cve-2026-63073/fixed.c" relpath = "crypto/cmp/cmp_vfy.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/cmp/cmp_vfy.c" +function = "ossl_cmp_msg_check_update" +mechanism = "other" sink = "unknown" evidence = "CMP unexpected sender DN used as format string in ERR_raise_data()" @@ -649,11 +749,15 @@ fixed = "openssl-cve-2026-14457/fixed.c" relpath = "ssl/t1_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/t1_lib.c" +function = "check_cert_usable" +mechanism = "other" sink = "unknown" evidence = "Handle signature_algorithms_cert extension in key-only context" @@ -674,11 +778,15 @@ fixed = "openssl-cve-2026-63074/fixed.c" relpath = "crypto/cmp/cmp_vfy.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/cmp/cmp_vfy.c" +function = "ossl_cmp_msg_check_update" +mechanism = "other" sink = "unknown" evidence = "Fix unbounded cert cache growth in cmp" @@ -699,11 +807,15 @@ fixed = "openssl-cve-2026-63075-txp_pkt_commit/fixed.c" relpath = "ssl/quic/quic_txp.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/quic/quic_txp.c" +function = "txp_pkt_commit" +mechanism = "other" sink = "unknown" evidence = "Don't store ACK-only frames in TX history for QUIC." @@ -724,11 +836,15 @@ fixed = "openssl-cve-2026-75803/fixed.c" relpath = "providers/implementations/ciphers/cipher_aes_ocb.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/ciphers/cipher_aes_ocb.c" +function = "aes_ocb_cipher" +mechanism = "other" sink = "unknown" evidence = "Check the tag on EVP_Cipher() finalize: Poly1305 and OCB AEADs" @@ -749,11 +865,15 @@ fixed = "openssl-cve-2026-75803-chacha20_poly1305_cipher/fixed.c" relpath = "providers/implementations/ciphers/cipher_chacha20_poly1305.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/ciphers/cipher_chacha20_poly1305.c" +function = "chacha20_poly1305_cipher" +mechanism = "other" sink = "unknown" evidence = "Check the tag on EVP_Cipher() finalize: Poly1305 and OCB AEADs" @@ -774,11 +894,15 @@ fixed = "openssl-cve-2026-75803-aes_gcm_siv_finish/fixed.c" relpath = "providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c" +function = "aes_gcm_siv_finish" +mechanism = "other" sink = "unknown" evidence = "Check the tag on EVP_Cipher() finalize: Poly1305 and OCB AEADs" @@ -799,11 +923,15 @@ fixed = "openssl-cve-2026-14456-port_default_packet_hand/fixed.c" relpath = "ssl/quic/quic_port.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/quic/quic_port.c" +function = "port_default_packet_handler" +mechanism = "other" sink = "unknown" evidence = "QUIC server: limit number of pending QUIC channels/connections" @@ -824,11 +952,15 @@ fixed = "openssl-cve-2026-14456-expect_quic_for_value/fixed.c" relpath = "ssl/quic/quic_impl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/quic/quic_impl.c" +function = "expect_quic_for_value" +mechanism = "other" sink = "unknown" evidence = "QUIC server: limit number of pending QUIC channels/connections" @@ -849,11 +981,15 @@ fixed = "openssl-cve-2026-45447/fixed.c" relpath = "crypto/pkcs7/pk7_smime.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-416" class = "use after free" path = "crypto/pkcs7/pk7_smime.c" +function = "PKCS7_verify" +mechanism = "uaf" sink = "free" evidence = "Fix possible use-after-free in OpenSSL PKCS7_verify()" @@ -874,11 +1010,15 @@ fixed = "openssl-cve-2026-45446/fixed.c" relpath = "providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c" +function = "aes_gcm_siv_initkey" +mechanism = "other" sink = "unknown" evidence = "Fix handling of empty-ciphertext messages in AES-GCM-SIV and AES-SIV" @@ -899,11 +1039,15 @@ fixed = "openssl-cve-2026-45446-aes_siv_set_ctx_params/fixed.c" relpath = "providers/implementations/ciphers/cipher_aes_siv.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/ciphers/cipher_aes_siv.c" +function = "aes_siv_set_ctx_params" +mechanism = "other" sink = "unknown" evidence = "Fix handling of empty-ciphertext messages in AES-GCM-SIV and AES-SIV" @@ -924,11 +1068,15 @@ fixed = "openssl-cve-2026-45445/fixed.c" relpath = "providers/implementations/ciphers/cipher_aes_ocb.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/ciphers/cipher_aes_ocb.c" +function = "aes_ocb_cipher" +mechanism = "other" sink = "unknown" evidence = "Apply the buffered IV on the AES-OCB EVP_Cipher() path" @@ -949,11 +1097,15 @@ fixed = "openssl-cve-2026-42771/fixed.c" relpath = "crypto/x509/x509_vpm.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/x509_vpm.c" +function = "validate_hostname_part" +mechanism = "other" sink = "unknown" evidence = "Fix length miscalculation in validate_email" @@ -974,11 +1126,15 @@ fixed = "openssl-cve-2026-42769/fixed.c" relpath = "crypto/cmp/cmp_genm.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/cmp/cmp_genm.c" +function = "selfsigned_verify_cb" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Use the correct issuer when validating rootCAKeyUpdate" @@ -999,11 +1155,15 @@ fixed = "openssl-cve-2026-42768/fixed.c" relpath = "crypto/cms/cms_env.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/cms/cms_env.c" +function = "cms_RecipientInfo_ktri_decrypt" +mechanism = "other" sink = "unknown" evidence = "Enforce implicit rejection for CMS/PKCS#7 decryption" @@ -1024,11 +1184,15 @@ fixed = "openssl-cve-2026-42768-pkcs7_decrypt_rinfo/fixed.c" relpath = "crypto/pkcs7/pk7_doit.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/pkcs7/pk7_doit.c" +function = "pkcs7_decrypt_rinfo" +mechanism = "other" sink = "unknown" evidence = "Enforce implicit rejection for CMS/PKCS#7 decryption" @@ -1049,11 +1213,15 @@ fixed = "openssl-cve-2026-42766/fixed.c" relpath = "crypto/cms/cms_pwri.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/cms/cms_pwri.c" +function = "ossl_cms_RecipientInfo_pwri_crypt" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix potential NULL dereference processing CMS PasswordRecipientInfo" @@ -1074,11 +1242,15 @@ fixed = "openssl-cve-2026-42765/fixed.c" relpath = "crypto/x509/x509_vfy.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/x509/x509_vfy.c" +function = "check_revocation" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix NULL Dereference in Certificate Verification with OCSP Checking" @@ -1099,11 +1271,15 @@ fixed = "openssl-cve-2026-42764/fixed.c" relpath = "ssl/quic/quic_port.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "ssl/quic/quic_port.c" +function = "port_default_packet_handler" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix NULL dereference in QUIC address validation" @@ -1124,11 +1300,15 @@ fixed = "openssl-cve-2026-35188/fixed.c" relpath = "crypto/x509/x509_vfy.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/x509_vfy.c" +function = "check_cert_ocsp_resp" +mechanism = "other" sink = "unknown" evidence = "Fix Double-free When Checking OCSP Stapled Response" @@ -1149,11 +1329,15 @@ fixed = "openssl-cve-2026-34181/fixed.c" relpath = "crypto/pkcs12/p12_mutl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/pkcs12/p12_mutl.c" +function = "PBMAC1_PBKDF2_HMAC" +mechanism = "other" sink = "unknown" evidence = "pkcs12: verify that the pbmac1 key length is safe" @@ -1174,11 +1358,15 @@ fixed = "openssl-cve-2026-9076/fixed.c" relpath = "crypto/cms/cms_pwri.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "crypto/cms/cms_pwri.c" +function = "kek_unwrap_key" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "cms: kek_unwrap_key: Fix out-of-bounds read in check-byte validation" @@ -1199,11 +1387,15 @@ fixed = "openssl-cve-2026-7383/fixed.c" relpath = "crypto/asn1/a_mbstr.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/asn1/a_mbstr.c" +function = "ASN1_mbstring_ncopy" +mechanism = "other" sink = "unknown" evidence = "Reject oversized inputs in ASN1_mbstring_ncopy()" @@ -1224,11 +1416,15 @@ fixed = "openssl-cve-2026-31790/fixed.c" relpath = "providers/implementations/kem/rsa_kem.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/kem/rsa_kem.c" +function = "rsasve_generate" +mechanism = "other" sink = "unknown" evidence = "rsa_kem: validate RSA_public_encrypt() result in RSASVE" @@ -1249,11 +1445,15 @@ fixed = "openssl-cve-2026-31790-rsasve_generate/fixed.in" relpath = "providers/implementations/kem/rsa_kem.c.in" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/kem/rsa_kem.c.in" +function = "rsasve_generate" +mechanism = "other" sink = "unknown" evidence = "rsa_kem: validate RSA_public_encrypt() result in RSASVE" @@ -1274,11 +1474,15 @@ fixed = "openssl-cve-2026-31789/fixed.c" relpath = "crypto/o_str.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "crypto/o_str.c" +function = "buf2hexstr_sep" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "Avoid possible buffer overflow in buf2hex conversion" @@ -1299,11 +1503,15 @@ fixed = "openssl-cve-2026-28388/fixed.c" relpath = "crypto/x509/x509_vfy.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/x509/x509_vfy.c" +function = "check_delta_base" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix NULL Dereference When Delta CRL Lacks CRL Number Extension" @@ -1324,11 +1532,15 @@ fixed = "openssl-cve-2026-28389/fixed.c" relpath = "crypto/cms/cms_rsa.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/cms/cms_rsa.c" +function = "rsa_cms_decrypt" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix NULL deref in rsa_cms_decrypt" @@ -1349,11 +1561,15 @@ fixed = "openssl-cve-2026-28389-dh_cms_set_shared_info/fixed.c" relpath = "crypto/cms/cms_dh.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/cms/cms_dh.c" +function = "dh_cms_set_shared_info" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix NULL deref in [ec]dh_cms_set_shared_info" @@ -1374,11 +1590,15 @@ fixed = "openssl-cve-2026-28389-ecdh_cms_set_shared_info/fixed.c" relpath = "crypto/cms/cms_ec.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/cms/cms_ec.c" +function = "ecdh_cms_set_shared_info" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix NULL deref in [ec]dh_cms_set_shared_info" @@ -1399,11 +1619,15 @@ fixed = "openssl-cve-2026-2673/fixed.c" relpath = "ssl/t1_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/t1_lib.c" +function = "gid_cb" +mechanism = "other" sink = "unknown" evidence = "Fix group tuple handling in DEFAULT expansion" @@ -1424,11 +1648,15 @@ fixed = "openssl-cve-2025-69419/fixed.c" relpath = "crypto/pkcs12/p12_utl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "crypto/pkcs12/p12_utl.c" +function = "bmp_to_utf8" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "Alternate fix for CVE-2025-69419" @@ -1449,11 +1677,15 @@ fixed = "openssl-cve-2025-69418/fixed.c" relpath = "crypto/modes/ocb128.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/modes/ocb128.c" +function = "CRYPTO_ocb128_encrypt" +mechanism = "other" sink = "unknown" evidence = "Fix OCB AES-NI/HW stream path unauthenticated/unencrypted trailing bytes" @@ -1474,11 +1706,15 @@ fixed = "openssl-cve-2025-66199-tls13_process_compressed/fixed.c" relpath = "ssl/statem/statem_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/statem_lib.c" +function = "tls13_process_compressed_certificate" +mechanism = "other" sink = "unknown" evidence = "Check the received uncompressed certificate length to prevent excessive pre-decompression allocation." @@ -1499,11 +1735,15 @@ fixed = "openssl-cve-2025-68160/fixed.c" relpath = "crypto/bio/bf_lbuf.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "crypto/bio/bf_lbuf.c" +function = "linebuffer_write" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "Fix heap buffer overflow in BIO_f_linebuffer" @@ -1524,11 +1764,15 @@ fixed = "openssl-cve-2025-15469/fixed.c" relpath = "apps/dgst.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "apps/dgst.c" +function = "do_fp_oneshot_sign" +mechanism = "other" sink = "unknown" evidence = "Report truncation in oneshot `openssl dgst -sign`" @@ -1549,11 +1793,15 @@ fixed = "openssl-cve-2025-15467/fixed.c" relpath = "crypto/evp/evp_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/evp/evp_lib.c" +function = "evp_cipher_get_asn1_aead_params" +mechanism = "other" sink = "unknown" evidence = "Correct handling of AEAD-encrypted CMS with inadmissibly long IV" @@ -1574,11 +1822,15 @@ fixed = "openssl-cve-2025-11187/fixed.c" relpath = "crypto/pkcs12/p12_mutl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/pkcs12/p12_mutl.c" +function = "PBMAC1_PBKDF2_HMAC" +mechanism = "other" sink = "unknown" evidence = "pkcs12: Validate salt and keylength in PBMAC1" @@ -1599,11 +1851,15 @@ fixed = "openssl-cve-2025-9232/fixed.c" relpath = "crypto/http/http_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/http/http_lib.c" +function = "use_proxy" +mechanism = "other" sink = "unknown" evidence = "use_proxy(): Add missing terminating NUL byte" @@ -1624,11 +1880,15 @@ fixed = "openssl-cve-2025-9230/fixed.c" relpath = "crypto/cms/cms_pwri.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "crypto/cms/cms_pwri.c" +function = "kek_unwrap_key" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "kek_unwrap_key(): Fix incorrect check of unwrapped key size" @@ -1649,11 +1909,15 @@ fixed = "openssl-cve-2024-12797/fixed.c" relpath = "ssl/statem/statem_clnt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/statem_clnt.c" +function = "tls_post_process_server_rpk" +mechanism = "other" sink = "unknown" evidence = "With SSL_VERIFY_PEER client RPK should abort on X509 error" @@ -1674,11 +1938,15 @@ fixed = "openssl-cve-2024-13176-ec_field_inverse_mod_ord/fixed.c" relpath = "crypto/ec/ec_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/ec/ec_lib.c" +function = "ec_field_inverse_mod_ord" +mechanism = "other" sink = "unknown" evidence = "Fix timing side-channel in ECDSA signature computation" @@ -1699,11 +1967,15 @@ fixed = "openssl-cve-2024-6119/fixed.c" relpath = "crypto/x509/v3_utl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/v3_utl.c" +function = "do_x509_check" +mechanism = "other" sink = "unknown" evidence = "Avoid type errors in EAI-related name check logic." @@ -1724,11 +1996,15 @@ fixed = "openssl-cve-2024-5535/fixed.c" relpath = "ssl/statem/extensions_clnt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/extensions_clnt.c" +function = "tls_parse_stoc_alpn" +mechanism = "other" sink = "unknown" evidence = "Add ALPN validation in the client" @@ -1749,11 +2025,15 @@ fixed = "openssl-cve-2024-5535-tls_construct_stoc_next_/fixed.c" relpath = "ssl/statem/extensions_srvr.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/extensions_srvr.c" +function = "tls_construct_stoc_next_proto_neg" +mechanism = "other" sink = "unknown" evidence = "Correct return values for tls_construct_stoc_next_proto_neg" @@ -1774,11 +2054,15 @@ fixed = "openssl-cve-2024-5535-tls_parse_stoc_npn/fixed.c" relpath = "ssl/statem/extensions_clnt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/extensions_clnt.c" +function = "tls_parse_stoc_npn" +mechanism = "other" sink = "unknown" evidence = "More correctly handle a selected_len of 0 when processing NPN" @@ -1799,11 +2083,15 @@ fixed = "openssl-cve-2024-5535-ssl_select_next_proto/fixed.c" relpath = "ssl/ssl_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/ssl_lib.c" +function = "SSL_select_next_proto" +mechanism = "other" sink = "unknown" evidence = "Fix SSL_select_next_proto" @@ -1824,11 +2112,15 @@ fixed = "openssl-cve-2024-5535-alpn_select_cb/fixed.c" relpath = "ssl/quic/quic_tserver.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/quic/quic_tserver.c" +function = "alpn_select_cb" +mechanism = "other" sink = "unknown" evidence = "Use correctly formatted ALPN data in tserver" @@ -1849,11 +2141,15 @@ fixed = "openssl-cve-2024-4741/fixed.c" relpath = "ssl/record/rec_layer_s3.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-416" class = "use after free" path = "ssl/record/rec_layer_s3.c" +function = "ssl3_read_n" +mechanism = "uaf" sink = "free" evidence = "Set rlayer.packet to NULL after we've finished using it" @@ -1874,11 +2170,15 @@ fixed = "openssl-cve-2024-4741-ssl3_release_read_buffer/fixed.c" relpath = "ssl/record/ssl3_buffer.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-416" class = "use after free" path = "ssl/record/ssl3_buffer.c" +function = "ssl3_release_read_buffer" +mechanism = "uaf" sink = "free" evidence = "Set rlayer.packet to NULL after we've finished using it" @@ -1899,11 +2199,15 @@ fixed = "openssl-cve-2024-4741-ssl_free_buffers/fixed.c" relpath = "ssl/ssl_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/ssl_lib.c" +function = "SSL_free_buffers" +mechanism = "other" sink = "unknown" evidence = "Only free the read buffers if we're not using them" @@ -1924,11 +2228,15 @@ fixed = "openssl-cve-2024-4741-tls_release_read_buffer/fixed.c" relpath = "ssl/record/methods/tls_common.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-416" class = "use after free" path = "ssl/record/methods/tls_common.c" +function = "tls_release_read_buffer" +mechanism = "uaf" sink = "free" evidence = "Set rl->packet to NULL after we've finished using it" @@ -1949,11 +2257,15 @@ fixed = "openssl-cve-2024-4741-tls_free_buffers/fixed.c" relpath = "ssl/record/methods/tls_common.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/record/methods/tls_common.c" +function = "tls_free_buffers" +mechanism = "other" sink = "unknown" evidence = "Only free the read buffers if we're not using them" @@ -1974,11 +2286,15 @@ fixed = "openssl-cve-2024-4603/fixed.c" relpath = "crypto/dsa/dsa_check.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/dsa/dsa_check.c" +function = "ossl_dsa_check_params" +mechanism = "other" sink = "unknown" evidence = "Check DSA parameters for excessive sizes before validating" @@ -1999,11 +2315,15 @@ fixed = "openssl-cve-2024-2511/fixed.c" relpath = "ssl/ssl_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/ssl_lib.c" +function = "ssl_update_cache" +mechanism = "other" sink = "unknown" evidence = "Fix unconstrained session cache growth in TLSv1.3" @@ -2024,11 +2344,15 @@ fixed = "openssl-cve-2024-2511-tls_construct_server_hel/fixed.c" relpath = "ssl/statem/statem_srvr.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/statem_srvr.c" +function = "tls_construct_server_hello" +mechanism = "other" sink = "unknown" evidence = "Fix unconstrained session cache growth in TLSv1.3" @@ -2049,11 +2373,15 @@ fixed = "openssl-cve-2024-0727/fixed.c" relpath = "crypto/pkcs12/p12_mutl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/pkcs12/p12_mutl.c" +function = "pkcs12_gen_mac" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Add NULL checks where ContentInfo data can be NULL" @@ -2074,11 +2402,15 @@ fixed = "openssl-cve-2024-0727-smime_write_pkcs7/fixed.c" relpath = "crypto/pkcs7/pk7_mime.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/pkcs7/pk7_mime.c" +function = "SMIME_write_PKCS7" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Add NULL checks where ContentInfo data can be NULL" @@ -2099,11 +2431,15 @@ fixed = "openssl-cve-2023-5363/fixed.c" relpath = "crypto/evp/evp_enc.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/evp/evp_enc.c" +function = "evp_cipher_init_internal" +mechanism = "other" sink = "unknown" evidence = "evp: process key length and iv length early if present" @@ -2124,11 +2460,15 @@ fixed = "openssl-cve-2023-3817-dh_check/fixed.c" relpath = "crypto/dh/dh_check.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/dh/dh_check.c" +function = "DH_check" +mechanism = "other" sink = "unknown" evidence = "DH_check(): Do not try checking q properties if it is obviously invalid" @@ -2149,11 +2489,15 @@ fixed = "openssl-cve-2023-3446/fixed.c" relpath = "crypto/dh/dh_check.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/dh/dh_check.c" +function = "DH_check" +mechanism = "other" sink = "unknown" evidence = "Fix DH_check() excessive time with over sized modulus" @@ -2174,11 +2518,15 @@ fixed = "openssl-cve-2023-2975/fixed.c" relpath = "providers/implementations/ciphers/cipher_aes_siv.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/ciphers/cipher_aes_siv.c" +function = "siv_cipher" +mechanism = "other" sink = "unknown" evidence = "Do not ignore empty associated data with AES-SIV mode" @@ -2199,11 +2547,15 @@ fixed = "openssl-cve-2023-2650/fixed.c" relpath = "crypto/objects/obj_dat.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/objects/obj_dat.c" +function = "OBJ_obj2txt" +mechanism = "other" sink = "unknown" evidence = "Restrict the size of OBJECT IDENTIFIERs that OBJ_obj2txt will translate" @@ -2224,11 +2576,15 @@ fixed = "openssl-cve-2022-37454/fixed.c" relpath = "providers/implementations/kdfs/hkdf.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "providers/implementations/kdfs/hkdf.c" +function = "HKDF_Expand" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "Update hkdf.c to avoid potentially vulnerable code pattern" @@ -2249,11 +2605,15 @@ fixed = "openssl-cve-2022-4304/fixed.c" relpath = "crypto/bn/bn_asm.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/bn/bn_asm.c" +function = "bn_sub_words" +mechanism = "other" sink = "unknown" evidence = "Alternative fix for CVE-2022-4304" @@ -2274,11 +2634,15 @@ fixed = "openssl-cve-2022-4304-bn_blinding_invert_ex/fixed.c" relpath = "crypto/bn/bn_blind.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/bn/bn_blind.c" +function = "BN_BLINDING_invert_ex" +mechanism = "other" sink = "unknown" evidence = "Alternative fix for CVE-2022-4304" @@ -2299,11 +2663,15 @@ fixed = "openssl-cve-2022-4304-rsa_blinding_invert/fixed.c" relpath = "crypto/rsa/rsa_ossl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/rsa/rsa_ossl.c" +function = "rsa_blinding_invert" +mechanism = "other" sink = "unknown" evidence = "Alternative fix for CVE-2022-4304" @@ -2324,11 +2692,15 @@ fixed = "openssl-cve-2023-0465/fixed.c" relpath = "crypto/x509/x509_vfy.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/x509_vfy.c" +function = "check_policy" +mechanism = "other" sink = "unknown" evidence = "Ensure that EXFLAG_INVALID_POLICY is checked even in leaf certs" @@ -2349,11 +2721,15 @@ fixed = "openssl-cve-2023-0464/fixed.c" relpath = "crypto/x509v3/pcy_tree.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509v3/pcy_tree.c" +function = "tree_init" +mechanism = "other" sink = "unknown" evidence = "x509: excessive resource use verifying policy constraints" @@ -2374,11 +2750,15 @@ fixed = "openssl-cve-2023-0464-tree_init/fixed.c" relpath = "crypto/x509/pcy_tree.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/pcy_tree.c" +function = "tree_init" +mechanism = "other" sink = "unknown" evidence = "x509: excessive resource use verifying policy constraints" @@ -2399,11 +2779,15 @@ fixed = "openssl-cve-2021-3449/fixed.c" relpath = "ssl/record/methods/tls_common.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/record/methods/tls_common.c" +function = "tls_get_more_records" +mechanism = "other" sink = "unknown" evidence = "fuzz: make post handshake reachable" @@ -2424,11 +2808,15 @@ fixed = "openssl-cve-2021-3449-def_generate_session_id/fixed.c" relpath = "ssl/ssl_sess.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/ssl_sess.c" +function = "def_generate_session_id" +mechanism = "other" sink = "unknown" evidence = "fuzz: make post handshake reachable" @@ -2449,11 +2837,15 @@ fixed = "openssl-cve-2021-3449-tls_parse_ctos_renegotia/fixed.c" relpath = "ssl/statem/extensions_srvr.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/extensions_srvr.c" +function = "tls_parse_ctos_renegotiate" +mechanism = "other" sink = "unknown" evidence = "fuzz: make post handshake reachable" @@ -2474,11 +2866,15 @@ fixed = "openssl-cve-2021-3449-tls_process_finished/fixed.c" relpath = "ssl/statem/statem_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/statem_lib.c" +function = "tls_process_finished" +mechanism = "other" sink = "unknown" evidence = "fuzz: make post handshake reachable" @@ -2499,11 +2895,15 @@ fixed = "openssl-cve-2023-0401/fixed.c" relpath = "crypto/pkcs7/pk7_doit.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/pkcs7/pk7_doit.c" +function = "pkcs7_bio_add_digest" +mechanism = "other" sink = "unknown" evidence = "pk7_doit.c: Check return of BIO_set_md() calls" @@ -2524,11 +2924,15 @@ fixed = "openssl-cve-2023-0286-general_name_cmp/fixed.c" relpath = "crypto/x509/v3_genn.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/v3_genn.c" +function = "GENERAL_NAME_cmp" +mechanism = "other" sink = "unknown" evidence = "Fix GENERAL_NAME_cmp for x400Address (master)" @@ -2549,11 +2953,15 @@ fixed = "openssl-cve-2023-0217/fixed.c" relpath = "crypto/ffc/ffc_key_validate.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/ffc/ffc_key_validate.c" +function = "ossl_ffc_validate_public_key_partial" +mechanism = "other" sink = "unknown" evidence = "Fix NULL deference when validating FFC public key." @@ -2574,11 +2982,15 @@ fixed = "openssl-cve-2023-0216/fixed.c" relpath = "crypto/pkcs7/pk7_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/pkcs7/pk7_lib.c" +function = "STACK_OF" +mechanism = "other" sink = "unknown" evidence = "Do not dereference PKCS7 object data if not set" @@ -2599,11 +3011,15 @@ fixed = "openssl-cve-2022-4304-rsa_ossl_private_decrypt/fixed.c" relpath = "crypto/rsa/rsa_ossl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/rsa/rsa_ossl.c" +function = "rsa_ossl_private_decrypt" +mechanism = "other" sink = "unknown" evidence = "Fix Timing Oracle in RSA decryption" @@ -2624,11 +3040,15 @@ fixed = "openssl-cve-2020-25659/fixed.c" relpath = "crypto/rsa/rsa_ossl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/rsa/rsa_ossl.c" +function = "rsa_ossl_private_decrypt" +mechanism = "other" sink = "unknown" evidence = "rsa: add implicit rejection in PKCS#1 v1.5" @@ -2649,11 +3069,15 @@ fixed = "openssl-cve-2022-3996/fixed.c" relpath = "crypto/x509/pcy_map.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/pcy_map.c" +function = "ossl_policy_cache_set_mapping" +mechanism = "other" sink = "unknown" evidence = "x509: fix double locking problem" @@ -2674,11 +3098,15 @@ fixed = "openssl-cve-2022-3786-ossl_punycode_decode/fixed.c" relpath = "crypto/punycode.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "crypto/punycode.c" +function = "ossl_punycode_decode" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "Fix CVE-2022-3786 in punycode decoder." @@ -2699,11 +3127,15 @@ fixed = "openssl-cve-2022-3602-ossl_punycode_decode/fixed.c" relpath = "crypto/punycode.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/punycode.c" +function = "ossl_punycode_decode" +mechanism = "other" sink = "unknown" evidence = "Fix CVE-2022-3602 in punycode decoder." @@ -2724,11 +3156,15 @@ fixed = "openssl-cve-2022-3358/fixed.c" relpath = "crypto/evp/digest.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/evp/digest.c" +function = "evp_md_init_internal" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix usage of custom EVP_CIPHER objects" @@ -2749,11 +3185,15 @@ fixed = "openssl-cve-2022-3358-evp_cipher_init_internal/fixed.c" relpath = "crypto/evp/evp_enc.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/evp/evp_enc.c" +function = "evp_cipher_init_internal" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix usage of custom EVP_CIPHER objects" @@ -2774,11 +3214,15 @@ fixed = "openssl-cve-2022-2068-hash_dir/fixed.in" relpath = "tools/c_rehash.in" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "tools/c_rehash.in" +function = "hash_dir" +mechanism = "other" sink = "unknown" evidence = "Fix file operations in c_rehash." @@ -2799,11 +3243,15 @@ fixed = "openssl-cve-2022-29242/fixed.c" relpath = "ssl/statem/statem_clnt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/statem_clnt.c" +function = "tls_construct_cke_gost18" +mechanism = "other" sink = "unknown" evidence = "Use safe pattern for buffer size determining in case of GOST key exchange" @@ -2824,11 +3272,15 @@ fixed = "openssl-cve-2022-1434/fixed.c" relpath = "providers/implementations/ciphers/cipher_rc4_hmac_md5.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/ciphers/cipher_rc4_hmac_md5.c" +function = "rc4_hmac_md5_set_ctx_params" +mechanism = "other" sink = "unknown" evidence = "Fix the RC4-MD5 cipher" @@ -2849,11 +3301,15 @@ fixed = "openssl-cve-2022-1343/fixed.c" relpath = "crypto/ocsp/ocsp_vfy.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/ocsp/ocsp_vfy.c" +function = "ocsp_verify_signer" +mechanism = "other" sink = "unknown" evidence = "Fix OCSP_basic_verify signer certificate validation" @@ -2874,11 +3330,15 @@ fixed = "openssl-cve-2021-4044/fixed.c" relpath = "crypto/x509/x509_vfy.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/x509_vfy.c" +function = "has_san_id" +mechanism = "other" sink = "unknown" evidence = "Fix infinite verification loops due to has_san_id" @@ -2899,11 +3359,15 @@ fixed = "openssl-cve-2021-4044-tls_post_process_server_/fixed.c" relpath = "ssl/statem/statem_clnt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/statem_clnt.c" +function = "tls_post_process_server_certificate" +mechanism = "other" sink = "unknown" evidence = "Fix invalid handling of verify errors in libssl" @@ -2924,11 +3388,15 @@ fixed = "openssl-cve-2021-3711-sm2_asym_decrypt/fixed.c" relpath = "providers/implementations/asymciphers/sm2_enc.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "providers/implementations/asymciphers/sm2_enc.c" +function = "sm2_asym_decrypt" +mechanism = "other" sink = "unknown" evidence = "Correctly calculate the length of SM2 plaintext given the ciphertext" @@ -2949,11 +3417,15 @@ fixed = "openssl-cve-2021-3712-netscape_spki_print/fixed.c" relpath = "crypto/asn1/t_spki.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/asn1/t_spki.c" +function = "NETSCAPE_SPKI_print" +mechanism = "other" sink = "unknown" evidence = "Fix NETSCAPE_SPKI_print function to not assume NUL terminated strings" @@ -2974,11 +3446,15 @@ fixed = "openssl-cve-2021-3712-append_ia5/fixed.c" relpath = "crypto/x509/v3_utl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/v3_utl.c" +function = "append_ia5" +mechanism = "other" sink = "unknown" evidence = "Fix append_ia5 function to not assume NUL terminated strings" @@ -2999,11 +3475,15 @@ fixed = "openssl-cve-2021-3712-ossl_cmp_hdr_push1_freet/fixed.c" relpath = "crypto/cmp/cmp_hdr.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/cmp/cmp_hdr.c" +function = "ossl_cmp_hdr_push1_freeText" +mechanism = "other" sink = "unknown" evidence = "Fix CMP code to not assume NUL terminated strings" @@ -3024,11 +3504,15 @@ fixed = "openssl-cve-2021-3712-ia5ncasecmp/fixed.c" relpath = "crypto/x509/v3_ncons.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/v3_ncons.c" +function = "ia5ncasecmp" +mechanism = "other" sink = "unknown" evidence = "Fix the name constraints code to not assume NUL terminated strings" @@ -3049,11 +3533,15 @@ fixed = "openssl-cve-2021-3712-i2r_pci/fixed.c" relpath = "crypto/x509/v3_pci.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/v3_pci.c" +function = "i2r_pci" +mechanism = "other" sink = "unknown" evidence = "Fix printing of PROXY_CERT_INFO_EXTENSION to not assume NUL terminated strings" @@ -3074,11 +3562,15 @@ fixed = "openssl-cve-2021-3712-general_name_print/fixed.c" relpath = "crypto/x509/v3_san.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/v3_san.c" +function = "GENERAL_NAME_print" +mechanism = "other" sink = "unknown" evidence = "Fix GENERAL_NAME_print to not assume NUL terminated strings" @@ -3099,11 +3591,15 @@ fixed = "openssl-cve-2021-3712-print_qualifiers/fixed.c" relpath = "crypto/x509/v3_cpols.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/v3_cpols.c" +function = "print_qualifiers" +mechanism = "other" sink = "unknown" evidence = "Fix POLICYINFO printing to not assume NUL terminated strings" @@ -3124,11 +3620,15 @@ fixed = "openssl-cve-2021-3711-pkey_sm2_decrypt/fixed.c" relpath = "crypto/sm2/sm2_pmeth.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/sm2/sm2_pmeth.c" +function = "pkey_sm2_decrypt" +mechanism = "other" sink = "unknown" evidence = "Correctly calculate the length of SM2 plaintext given the ciphertext" @@ -3149,11 +3649,15 @@ fixed = "openssl-cve-2021-3449-ssl_generate_master_secr/fixed.c" relpath = "ssl/s3_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/s3_lib.c" +function = "ssl_generate_master_secret" +mechanism = "other" sink = "unknown" evidence = "Ensure buffer/length pairs are always in sync" @@ -3174,11 +3678,15 @@ fixed = "openssl-cve-2021-3449-ssl_ctx_set_alpn_protos/fixed.c" relpath = "ssl/ssl_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/ssl_lib.c" +function = "SSL_CTX_set_alpn_protos" +mechanism = "other" sink = "unknown" evidence = "Ensure buffer/length pairs are always in sync" @@ -3199,11 +3707,15 @@ fixed = "openssl-cve-2021-3449-init_sig_algs_cert/fixed.c" relpath = "ssl/statem/extensions.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/extensions.c" +function = "init_sig_algs_cert" +mechanism = "other" sink = "unknown" evidence = "Ensure buffer/length pairs are always in sync" @@ -3224,11 +3736,15 @@ fixed = "openssl-cve-2021-3449-tls_construct_ctos_early/fixed.c" relpath = "ssl/statem/extensions_clnt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/extensions_clnt.c" +function = "tls_construct_ctos_early_data" +mechanism = "other" sink = "unknown" evidence = "Ensure buffer/length pairs are always in sync" @@ -3249,11 +3765,15 @@ fixed = "openssl-cve-2021-3449-tls_process_certificate_/fixed.c" relpath = "ssl/statem/statem_clnt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/statem_clnt.c" +function = "tls_process_certificate_request" +mechanism = "other" sink = "unknown" evidence = "Ensure buffer/length pairs are always in sync" @@ -3274,11 +3794,15 @@ fixed = "openssl-cve-2021-3449-tls_handle_alpn/fixed.c" relpath = "ssl/statem/statem_srvr.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "ssl/statem/statem_srvr.c" +function = "tls_handle_alpn" +mechanism = "other" sink = "unknown" evidence = "Ensure buffer/length pairs are always in sync" @@ -3299,11 +3823,15 @@ fixed = "openssl-cve-2021-3449-init_sig_algs/fixed.c" relpath = "ssl/statem/extensions.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "ssl/statem/extensions.c" +function = "init_sig_algs" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "ssl sigalg extension: fix NULL pointer dereference" @@ -3324,11 +3852,15 @@ fixed = "openssl-cve-2021-3450/fixed.c" relpath = "crypto/x509/x509_vfy.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509/x509_vfy.c" +function = "check_chain_extensions" +mechanism = "other" sink = "unknown" evidence = "check_chain_extensions: Do not override error return value by check_curve" @@ -3349,11 +3881,15 @@ fixed = "openssl-cve-2021-23840-evp_encryptdecryptupdate/fixed.c" relpath = "crypto/evp/evp_enc.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "buffer overflow" path = "crypto/evp/evp_enc.c" +function = "evp_EncryptDecryptUpdate" +mechanism = "unchecked_length_copy" sink = "memcpy" evidence = "Don't overflow the output length in EVP_CipherUpdate calls" @@ -3374,11 +3910,15 @@ fixed = "openssl-cve-2021-23841/fixed.c" relpath = "crypto/x509/x509_cmp.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "crypto/x509/x509_cmp.c" +function = "X509_issuer_and_serial_hash" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix Null pointer deref in X509_issuer_and_serial_hash()" @@ -3399,11 +3939,15 @@ fixed = "openssl-cve-2020-1971-general_name_cmp/fixed.c" relpath = "crypto/x509v3/v3_genn.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "crypto/x509v3/v3_genn.c" +function = "GENERAL_NAME_cmp" +mechanism = "other" sink = "unknown" evidence = "Correctly compare EdiPartyName in GENERAL_NAME_cmp()" @@ -3424,11 +3968,15 @@ fixed = "openssl-cve-2020-1967-tls1_check_sig_alg/fixed.c" relpath = "ssl/t1_lib.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-476" class = "null pointer dereference" path = "ssl/t1_lib.c" +function = "tls1_check_sig_alg" +mechanism = "null_deref_unguarded" sink = "NULL" evidence = "Fix NULL dereference in SSL_check_chain() for TLS 1.3" @@ -3449,11 +3997,15 @@ fixed = "curl-cve-2026-9080/fixed.c" relpath = "lib/multi_ev.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/multi_ev.c" +function = "mev_forget_socket" +mechanism = "other" sink = "unknown" evidence = "multi_ev: refresh sock entry after remove callback" @@ -3474,11 +4026,15 @@ fixed = "curl-cve-2023-38545-do_socks5/fixed.c" relpath = "lib/socks.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/socks.c" +function = "do_SOCKS5" +mechanism = "other" sink = "unknown" evidence = "socks: return error if hostname too long for remote resolve" @@ -3499,11 +4055,15 @@ fixed = "curl-cve-2022-32207/fixed.c" relpath = "lib/cookie.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/cookie.c" +function = "cookie_output" +mechanism = "other" sink = "unknown" evidence = "fopen: add Curl_fopen() for better overwriting of files" @@ -3524,11 +4084,15 @@ fixed = "curl-cve-2022-32206/fixed.c" relpath = "lib/content_encoding.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/content_encoding.c" +function = "Curl_build_unencoding_stack" +mechanism = "other" sink = "unknown" evidence = "content_encoding: return error on too many compression steps" @@ -3549,11 +4113,15 @@ fixed = "curl-cve-2022-32208/fixed.c" relpath = "lib/krb5.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/krb5.c" +function = "read_data" +mechanism = "other" sink = "unknown" evidence = "krb5: return error properly on decode errors" @@ -3574,11 +4142,15 @@ fixed = "curl-cve-2022-27782-curl_init_userdefined/fixed.c" relpath = "lib/url.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/url.c" +function = "Curl_init_userdefined" +mechanism = "other" sink = "unknown" evidence = "tls: check more TLS details for connection reuse" @@ -3599,11 +4171,15 @@ fixed = "curl-cve-2022-27782-gtls_shutdown/fixed.c" relpath = "lib/vtls/gtls.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/vtls/gtls.c" +function = "gtls_shutdown" +mechanism = "other" sink = "unknown" evidence = "tls: check more TLS details for connection reuse" @@ -3624,11 +4200,15 @@ fixed = "curl-cve-2022-27782-nss_setup_connect/fixed.c" relpath = "lib/vtls/nss.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/vtls/nss.c" +function = "nss_setup_connect" +mechanism = "other" sink = "unknown" evidence = "tls: check more TLS details for connection reuse" @@ -3649,11 +4229,15 @@ fixed = "curl-cve-2022-27782-ossl_connect_step1/fixed.c" relpath = "lib/vtls/openssl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/vtls/openssl.c" +function = "ossl_connect_step1" +mechanism = "other" sink = "unknown" evidence = "tls: check more TLS details for connection reuse" @@ -3674,11 +4258,15 @@ fixed = "curl-cve-2022-27782-curl_free_primary_ssl_co/fixed.c" relpath = "lib/vtls/vtls.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/vtls/vtls.c" +function = "Curl_free_primary_ssl_config" +mechanism = "other" sink = "unknown" evidence = "tls: check more TLS details for connection reuse" @@ -3699,11 +4287,15 @@ fixed = "curl-cve-2022-27778/fixed.c" relpath = "src/tool_operate.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "src/tool_operate.c" +function = "post_per_transfer" +mechanism = "other" sink = "unknown" evidence = "post_per_transfer: remove the updated file name" @@ -3724,11 +4316,15 @@ fixed = "curl-cve-2022-30115/fixed.c" relpath = "lib/hsts.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/hsts.c" +function = "hsts_create" +mechanism = "other" sink = "unknown" evidence = "hsts: ignore trailing dots when comparing hosts names" @@ -3749,11 +4345,15 @@ fixed = "curl-cve-2022-27780/fixed.c" relpath = "lib/urlapi.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/urlapi.c" +function = "hostname_check" +mechanism = "other" sink = "unknown" evidence = "urlapi: reject percent-decoding host name into separator bytes" @@ -3774,11 +4374,15 @@ fixed = "curl-cve-2022-27774/fixed.c" relpath = "lib/transfer.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/transfer.c" +function = "Curl_follow" +mechanism = "other" sink = "unknown" evidence = "transfer: redirects to other protocols or ports clear auth" @@ -3799,11 +4403,15 @@ fixed = "curl-cve-2022-27776/fixed.c" relpath = "lib/http.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/http.c" +function = "Curl_add_custom_headers" +mechanism = "other" sink = "unknown" evidence = "http: avoid auth/cookie on redirects same host diff port" @@ -3824,11 +4432,15 @@ fixed = "curl-cve-2022-27775/fixed.c" relpath = "lib/conncache.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/conncache.c" +function = "hashkey" +mechanism = "other" sink = "unknown" evidence = "conncache: include the zone id in the \"bundle\" hashkey" @@ -3849,11 +4461,15 @@ fixed = "curl-cve-2022-22576-conn_free/fixed.c" relpath = "lib/url.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/url.c" +function = "conn_free" +mechanism = "other" sink = "unknown" evidence = "url: check sasl additional parameters for connection reuse." @@ -3874,11 +4490,15 @@ fixed = "curl-cve-2021-22947/fixed.c" relpath = "lib/ftp.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/ftp.c" +function = "ftp_statemachine" +mechanism = "other" sink = "unknown" evidence = "ftp,imap,pop3,smtp: reject STARTTLS server response pipelining" @@ -3899,11 +4519,15 @@ fixed = "curl-cve-2021-22947-imap_state_starttls_resp/fixed.c" relpath = "lib/imap.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/imap.c" +function = "imap_state_starttls_resp" +mechanism = "other" sink = "unknown" evidence = "ftp,imap,pop3,smtp: reject STARTTLS server response pipelining" @@ -3924,11 +4548,15 @@ fixed = "curl-cve-2021-22947-pop3_state_starttls_resp/fixed.c" relpath = "lib/pop3.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/pop3.c" +function = "pop3_state_starttls_resp" +mechanism = "other" sink = "unknown" evidence = "ftp,imap,pop3,smtp: reject STARTTLS server response pipelining" @@ -3949,11 +4577,15 @@ fixed = "curl-cve-2021-22947-smtp_state_starttls_resp/fixed.c" relpath = "lib/smtp.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/smtp.c" +function = "smtp_state_starttls_resp" +mechanism = "other" sink = "unknown" evidence = "ftp,imap,pop3,smtp: reject STARTTLS server response pipelining" @@ -3974,11 +4606,15 @@ fixed = "curl-cve-2021-22946/fixed.c" relpath = "lib/ftp.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/ftp.c" +function = "ftp_statemachine" +mechanism = "other" sink = "unknown" evidence = "ftp,imap,pop3: do not ignore --ssl-reqd" @@ -3999,11 +4635,15 @@ fixed = "curl-cve-2021-22946-imap_state_capability_re/fixed.c" relpath = "lib/imap.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/imap.c" +function = "imap_state_capability_resp" +mechanism = "other" sink = "unknown" evidence = "ftp,imap,pop3: do not ignore --ssl-reqd" @@ -4024,11 +4664,15 @@ fixed = "curl-cve-2021-22946-pop3_state_capa_resp/fixed.c" relpath = "lib/pop3.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/pop3.c" +function = "pop3_state_capa_resp" +mechanism = "other" sink = "unknown" evidence = "ftp,imap,pop3: do not ignore --ssl-reqd" @@ -4049,11 +4693,15 @@ fixed = "curl-cve-2021-22945/fixed.c" relpath = "lib/mqtt.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/mqtt.c" +function = "mqtt_send" +mechanism = "other" sink = "unknown" evidence = "mqtt: clear the leftovers pointer when sending succeeds" @@ -4074,11 +4722,15 @@ fixed = "curl-cve-2021-22924/fixed.c" relpath = "lib/url.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/url.c" +function = "create_conn" +mechanism = "other" sink = "unknown" evidence = "vtls: fix connection reuse checks for issuer cert and case sensitivity" @@ -4099,11 +4751,15 @@ fixed = "curl-cve-2021-22924-nss_do_connect/fixed.c" relpath = "lib/vtls/nss.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/vtls/nss.c" +function = "nss_do_connect" +mechanism = "other" sink = "unknown" evidence = "vtls: fix connection reuse checks for issuer cert and case sensitivity" @@ -4124,11 +4780,15 @@ fixed = "curl-cve-2021-22924-servercert/fixed.c" relpath = "lib/vtls/openssl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/vtls/openssl.c" +function = "servercert" +mechanism = "other" sink = "unknown" evidence = "vtls: fix connection reuse checks for issuer cert and case sensitivity" @@ -4149,11 +4809,15 @@ fixed = "curl-cve-2021-22925/fixed.c" relpath = "lib/telnet.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/telnet.c" +function = "suboption" +mechanism = "other" sink = "unknown" evidence = "telnet: fix option parser to not send uninitialized contents" @@ -4174,11 +4838,15 @@ fixed = "curl-cve-2021-22901-ossl_connect_step1/fixed.c" relpath = "lib/vtls/openssl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/vtls/openssl.c" +function = "ossl_connect_step1" +mechanism = "other" sink = "unknown" evidence = "openssl: associate/detach the transfer from connection" @@ -4199,11 +4867,15 @@ fixed = "curl-cve-2021-22898/fixed.c" relpath = "lib/telnet.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/telnet.c" +function = "suboption" +mechanism = "other" sink = "unknown" evidence = "telnet: check sscanf() for correct number of matches" @@ -4224,11 +4896,15 @@ fixed = "curl-cve-2021-22890/fixed.c" relpath = "lib/vtls/bearssl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/vtls/bearssl.c" +function = "bearssl_connect_step1" +mechanism = "other" sink = "unknown" evidence = "vtls: add 'isproxy' argument to Curl_ssl_get/addsessionid()" @@ -4249,11 +4925,15 @@ fixed = "curl-cve-2021-22876/fixed.c" relpath = "lib/transfer.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/transfer.c" +function = "Curl_follow" +mechanism = "other" sink = "unknown" evidence = "transfer: strip credentials from the auto-referer header field" @@ -4274,11 +4954,15 @@ fixed = "curl-cve-2020-8286/fixed.c" relpath = "lib/vtls/openssl.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/vtls/openssl.c" +function = "verifystatus" +mechanism = "other" sink = "unknown" evidence = "openssl: make the OCSP verification verify the certificate id" @@ -4299,11 +4983,15 @@ fixed = "curl-cve-2020-8284/fixed.c" relpath = "lib/url.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/url.c" +function = "Curl_init_userdefined" +mechanism = "other" sink = "unknown" evidence = "ftp: CURLOPT_FTP_SKIP_PASV_IP by default" @@ -4324,11 +5012,15 @@ fixed = "curl-cve-2020-8284-config_init/fixed.c" relpath = "src/tool_cfgable.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "src/tool_cfgable.c" +function = "config_init" +mechanism = "other" sink = "unknown" evidence = "ftp: CURLOPT_FTP_SKIP_PASV_IP by default" @@ -4349,11 +5041,15 @@ fixed = "curl-cve-2020-8231-curl_multi_add_handle/fixed.c" relpath = "lib/multi.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "holdout" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/multi.c" +function = "curl_multi_add_handle" +mechanism = "other" sink = "unknown" evidence = "Curl_easy: remember last connection by id, not by pointer" @@ -4374,11 +5070,15 @@ fixed = "curl-cve-2020-8231-curl_open/fixed.c" relpath = "lib/url.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "lib/url.c" +function = "Curl_open" +mechanism = "other" sink = "unknown" evidence = "Curl_easy: remember last connection by id, not by pointer" @@ -4399,10 +5099,14 @@ fixed = "curl-cve-2020-8177/fixed.c" relpath = "src/tool_cb_hdr.c" min_recall = 1.0 fix_policy = "silent" +provenance = "human" +split = "train" [[pair.expected]] cwe = "CWE-119" class = "security fix" path = "src/tool_cb_hdr.c" +function = "tool_header_cb" +mechanism = "other" sink = "unknown" evidence = "tool_getparam: -i is not OK if -J is used" diff --git a/src/openultrasast/benchmark.py b/src/openultrasast/benchmark.py index bef8418..88eca9c 100644 --- a/src/openultrasast/benchmark.py +++ b/src/openultrasast/benchmark.py @@ -26,6 +26,7 @@ class ExpectedFinding: line: int | None = None function: str | None = None sink: str | None = None + mechanism: str | None = None @dataclass(frozen=True) @@ -161,6 +162,7 @@ def load_benchmark_manifest(path: Path) -> BenchmarkManifest: line=int(item["line"]) if "line" in item else None, function=str(item["function"]) if "function" in item else None, sink=str(item["sink"]) if "sink" in item else None, + mechanism=str(item["mechanism"]) if "mechanism" in item else None, ) for item in expected ], diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index f4c5ddc..87dd90c 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -41,7 +41,17 @@ from .improve import RoundOutcome, run_improvement from .index import build_code_chunks from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints from .pair_gate import print_pair_metrics -from .pairs import DEFAULT_CATALOG, evaluate_catalog, load_pair_catalog, result_payload, select_slice +from .pairs import ( + DEFAULT_CATALOG, + SLICE_NAMES, + evaluate_catalog, + load_pair_catalog, + make_hunter_scan, + result_payload, + select_profile, + select_slice, + select_split, +) from .policy import assert_rules_resolve, load_policy from .preprocess import preprocess_repository, write_preprocess_artifact from .provider.openrouter import OpenRouterEmbeddingClient, OpenRouterError @@ -125,7 +135,13 @@ def main(argv: list[str] | None = None) -> int: help="scan isolated vuln-vs-fixed pairs (TP on vuln, silent on fix) and emit improve-loop signals", ) pairs.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) - pairs.add_argument("--slice", choices=("all", "local", "github", "sast", "vfc"), default="all") + pairs.add_argument("--slice", choices=SLICE_NAMES, default="all") + pairs.add_argument( + "--profile", choices=("all", "human", "agent", "mixed", "synthetic"), default="all", help="filter pairs by provenance profile" + ) + pairs.add_argument("--split", choices=("all", "train", "holdout"), default="all", help="filter pairs by declared split") + pairs.add_argument("--hunter", action="store_true", help="also score the LLM tool hunter on overlay slices (needs a hunter model)") + pairs.add_argument("--hunter-model", default=None, help="model id for --hunter (default: [models].hunter from openultrasast.toml)") pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON") subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration") @@ -149,7 +165,15 @@ def main(argv: list[str] | None = None) -> int: dry_run=args.dry_run, ) if args.command == "pairs": - return _pairs(args.catalog, args.slice, json_out=args.json) + return _pairs( + args.catalog, + args.slice, + json_out=args.json, + profile=args.profile, + split=args.split, + hunter=args.hunter, + hunter_model=args.hunter_model, + ) if args.command == "mcp": from .mcp import serve # lazy: keeps the import cycle (mcp -> cli) one-directional @@ -724,11 +748,28 @@ def _improve( return 0 -def _pairs(catalog: Path, slice_name: str, *, json_out: bool) -> int: +def _pairs( + catalog: Path, + slice_name: str, + *, + json_out: bool, + profile: str = "all", + split: str = "all", + hunter: bool = False, + hunter_model: str | None = None, +) -> int: if not catalog.exists() or not catalog.is_file(): raise SystemExit(f"pair catalog is not a file: {catalog}") - cases = select_slice(load_pair_catalog(catalog), slice_name) - result = evaluate_catalog(cases) + cases = select_split(select_profile(select_slice(load_pair_catalog(catalog), slice_name), profile), split) + scan = None + if hunter: + config_path = Path("openultrasast.toml") + config = load_config(config_path if config_path.exists() else None) + model = hunter_model or config.models.hunter or "" + client = tool_hunter.resolve_hunter_client() + if model and client is not None: + scan = make_hunter_scan(client, model) + result = evaluate_catalog(cases, hunter=scan) if json_out: print(json.dumps(result_payload(result), indent=2, sort_keys=True)) return 0 diff --git a/src/openultrasast/pairs.py b/src/openultrasast/pairs.py index b1fabf2..ed33113 100644 --- a/src/openultrasast/pairs.py +++ b/src/openultrasast/pairs.py @@ -3,6 +3,12 @@ Isolated trees keep the two sides from contaminating each other (a `safe.py` sitting next to `app.py` cannot measure false positives). Pair outcomes feed the improvement loop as miss (vuln) and fp (fix) signals. + +pair-corpus-honesty: the overlay scorer counts ``coverage`` records with a +source as detections and as leaks, matches by function when a label names one, +flags CWE-only rows as weak labels, reports parser/adjudication loss, and +slices metrics by provenance profile and by mechanism. Known-limit pairs are +evaluated but excluded from the achievable numbers. """ from __future__ import annotations @@ -10,8 +16,9 @@ from __future__ import annotations import shutil import tempfile import tomllib -from collections.abc import Sequence -from dataclasses import asdict, dataclass, field +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, field, replace +from functools import lru_cache from pathlib import Path from .benchmark import ( @@ -24,15 +31,38 @@ from .benchmark import ( ) from .findings import StaticFinding, quick_scan_findings from .mapping import analyze_entry_points, attach_reachability_hints -from .preprocess import preprocess_repository +from .preprocess import FileTarget, preprocess_repository from .rank import rank_targets +from .ruleset import PatternRule from .semantic import OverlayRecord, adjudicate +from .semantic.facts import FactLoadError, load_facts +from .semantic.functions import RESERVED_FUNCTION_LABELS, function_at, named_function_ranges, spans_named DEFAULT_CATALOG = Path("benchmarks/pairs/catalog.toml") DEFAULT_SAST_CATALOG = Path("benchmarks/pairs/sast/catalog.toml") DEFAULT_VFC_CATALOG = Path("benchmarks/pairs/vfc/catalog.toml") +DEFAULT_VIBE_PY_CATALOG = Path("benchmarks/pairs/vibe-py/catalog.toml") +DEFAULT_VFC_JS_CATALOG = Path("benchmarks/pairs/vfc-js/catalog.toml") +DEFAULT_AGENT_VFC_CATALOG = Path("benchmarks/pairs/agent-vfc/catalog.toml") DEFAULT_DATASETS = Path("benchmarks/pairs/datasets.toml") -OVERLAY_SLICES = frozenset({"sast", "vfc"}) +DEFAULT_MECHANISMS = Path("benchmarks/pairs/mechanisms.toml") + +# Slices scored on overlay dispositions (inventory fallback when nothing adjudicates). +OVERLAY_SLICES = frozenset({"sast", "vfc", "vibe-py", "vfc-js", "agent-vfc"}) +# Harvested function slices: every label must name the function (design §LabelSchema); "" does not scope. +FUNCTION_REQUIRED_SLICES = OVERLAY_SLICES - {"sast"} +FunctionRanges = dict[str, tuple[tuple[str, int, int], ...]] +SLICE_NAMES = ("all", "local", "github", "sast", "vfc", "vibe-py", "vfc-js", "agent-vfc") +PROVENANCES = frozenset({"human", "agent", "mixed", "synthetic"}) +SPLITS = frozenset({"train", "holdout"}) +_DEFAULT_PROVENANCE = {"sast": "synthetic", "local": "human", "github": "human", "vfc": "human"} +_MANIFEST_MECHANISM = "source_reaches_sink" # cheat-sheet fixtures referenced via expected_from + +HunterScan = Callable[[Path], list[StaticFinding]] + + +class CatalogError(ValueError): + """Raised when a pair catalog violates the label schema.""" @dataclass(frozen=True) @@ -52,6 +82,9 @@ class PairCase: commit_url: str = "" cve: str = "" license: str = "" + provenance: str = "human" + split: str = "train" + known_limit: str | None = None @dataclass(frozen=True) @@ -73,6 +106,17 @@ class PairOutcome: leaks: tuple[str, ...] commit_url: str = "" cve: str = "" + provenance: str = "human" + mechanisms: tuple[str, ...] = () + known_limit: str | None = None + split: str = "train" + weak_labels: int = 0 + parse_failed_vuln: int = 0 + parse_failed_fixed: int = 0 + unadjudicated_vuln: int = 0 + unadjudicated_fixed: int = 0 + detection_kinds: tuple[str, ...] = () + unresolved_labels: int = 0 # expected rows whose function names no range on the vulnerable side @dataclass(frozen=True) @@ -98,6 +142,12 @@ class PairEvalResult: per_slice: dict[str, PairCorpusMetrics] signals: tuple[dict[str, object], ...] scorers: dict[str, dict[str, PairCorpusMetrics]] = field(default_factory=dict) + per_profile: dict[str, PairCorpusMetrics] = field(default_factory=dict) + per_mechanism: dict[str, PairCorpusMetrics] = field(default_factory=dict) + achievable: dict[str, PairCorpusMetrics] = field(default_factory=dict) + known_limit: tuple[str, ...] = () + loss: dict[str, dict[str, int]] = field(default_factory=dict) + degradations: tuple[dict[str, object], ...] = () def load_datasets(path: Path = DEFAULT_DATASETS) -> tuple[dict[str, object], ...]: @@ -105,13 +155,30 @@ def load_datasets(path: Path = DEFAULT_DATASETS) -> tuple[dict[str, object], ... return tuple(dict(item) for item in payload.get("dataset", [])) +@lru_cache(maxsize=4) +def load_mechanisms(path: Path = DEFAULT_MECHANISMS) -> tuple[str, frozenset[str]]: + """Closed mechanism vocabulary: (version, ids). Missing file is a catalog error.""" + if not path.is_file(): + raise CatalogError(f"mechanism vocabulary missing: {path}") + payload = tomllib.loads(path.read_text()) + ids = frozenset(str(item["id"]) for item in payload.get("mechanism", []) if isinstance(item, dict) and item.get("id")) + if not ids: + raise CatalogError(f"mechanism vocabulary is empty: {path}") + return str(payload.get("version", "1")), ids + + def load_pair_catalog(path: Path = DEFAULT_CATALOG) -> tuple[PairCase, ...]: cases = _load_catalog_file(path) if path.resolve() == DEFAULT_CATALOG.resolve(): - if DEFAULT_SAST_CATALOG.exists(): - cases = cases + _load_catalog_file(DEFAULT_SAST_CATALOG) - if DEFAULT_VFC_CATALOG.exists(): - cases = cases + _load_catalog_file(DEFAULT_VFC_CATALOG) + for extra in ( + DEFAULT_SAST_CATALOG, + DEFAULT_VFC_CATALOG, + DEFAULT_VIBE_PY_CATALOG, + DEFAULT_VFC_JS_CATALOG, + DEFAULT_AGENT_VFC_CATALOG, + ): + if extra.exists(): + cases = cases + _load_catalog_file(extra) return cases @@ -119,13 +186,29 @@ def _load_catalog_file(path: Path) -> tuple[PairCase, ...]: catalog_path = path.resolve() payload = tomllib.loads(catalog_path.read_text()) root = catalog_path.parent + _, mechanisms = load_mechanisms() cases: list[PairCase] = [] for item in payload.get("pair", []): - expected = _expected_for(item, root) + name = str(item["name"]) + slice_name = str(item.get("slice", "github")) + expected = _expected_for(item, root, name, mechanisms) + if slice_name in FUNCTION_REQUIRED_SLICES: + for row in expected: + if not row.function or row.function in RESERVED_FUNCTION_LABELS: + raise CatalogError( + f"pair {name}: slice {slice_name} requires a named function on every expected row (got {row.function!r})" + ) + provenance = str(item.get("provenance") or _DEFAULT_PROVENANCE.get(slice_name, "human")) + if provenance not in PROVENANCES: + raise CatalogError(f"pair {name}: unknown provenance {provenance!r}; expected one of {sorted(PROVENANCES)}") + split = str(item.get("split", "train")) + if split not in SPLITS: + raise CatalogError(f"pair {name}: unknown split {split!r}; expected train or holdout") + known_limit = item.get("known_limit") cases.append( PairCase( - name=str(item["name"]), - slice=str(item.get("slice", "github")), + name=name, + slice=slice_name, language=str(item.get("language", "")), origin=str(item.get("origin", "")), vuln_file=_resolve(root, str(item["vuln"])), @@ -139,30 +222,50 @@ def _load_catalog_file(path: Path) -> tuple[PairCase, ...]: commit_url=str(item.get("commit_url", "")), cve=str(item.get("cve", "")), license=str(item.get("license", "")), + provenance=provenance, + split=split, + known_limit=str(known_limit) if known_limit else None, ) ) return tuple(cases) -def evaluate_pair(case: PairCase) -> PairOutcome: +def evaluate_pair(case: PairCase, *, hunter: HunterScan | None = None, ruleset: tuple[PatternRule, ...] | None = None) -> PairOutcome: with tempfile.TemporaryDirectory(prefix="ousast-pair-") as scratch: vuln_root = _materialize(Path(scratch) / "vuln", case.vuln_file, case.relpath) fix_root = _materialize(Path(scratch) / "fixed", case.fixed_file, case.relpath) + if hunter is not None: + return _evaluate_hunter_pair(case, vuln_root, fix_root, hunter) if case.slice in OVERLAY_SLICES: - return _evaluate_overlay_pair(case, vuln_root, fix_root) - return _evaluate_inventory_pair(case, vuln_root, fix_root) + return _evaluate_overlay_pair(case, vuln_root, fix_root, ruleset) + return _evaluate_inventory_pair(case, vuln_root, fix_root, ruleset=ruleset) -def evaluate_catalog(cases: Sequence[PairCase]) -> PairEvalResult: - outcomes = tuple(evaluate_pair(case) for case in cases) +def evaluate_catalog( + cases: Sequence[PairCase], + *, + hunter: HunterScan | None = None, + ruleset: tuple[PatternRule, ...] | None = None, + inventory_sidecar: bool = True, +) -> PairEvalResult: + """Score every case. ``ruleset`` lets the improve loop score a candidate ledger; ``inventory_sidecar=False`` skips the second scan.""" + outcomes = tuple(evaluate_pair(case, ruleset=ruleset) for case in cases) scorers: dict[str, dict[str, PairCorpusMetrics]] = {} - vfc_cases = tuple(case for case in cases if case.slice == "vfc") - if vfc_cases: - overlay_outcomes = tuple(item for item in outcomes if item.slice == "vfc") - scorers["vfc"] = { - "overlay": _metrics(overlay_outcomes), - "inventory": _metrics(tuple(_inventory_only(case) for case in vfc_cases)), - } + degradations: list[dict[str, object]] = [] + for slice_name in sorted({case.slice for case in cases if case.slice in OVERLAY_SLICES}): + slice_cases = tuple(case for case in cases if case.slice == slice_name) + scorers[slice_name] = {"overlay": _metrics(tuple(item for item in outcomes if item.slice == slice_name))} + if inventory_sidecar: + scorers[slice_name]["inventory"] = _metrics(tuple(_inventory_only(case, ruleset) for case in slice_cases)) + if hunter is not None: + scorers[slice_name]["hunter"] = _metrics(tuple(evaluate_pair(case, hunter=hunter) for case in slice_cases)) + if hunter is None and scorers: + degradations.append({"stage": "pairs", "reason": "hunter_model_unavailable"}) + achievable_outcomes = tuple(item for item in outcomes if item.known_limit is None) + per_mechanism: dict[str, list[PairOutcome]] = {} + for item in outcomes: + for mechanism in item.mechanisms: + per_mechanism.setdefault(mechanism, []).append(item) return PairEvalResult( outcomes=outcomes, overall=_metrics(outcomes), @@ -172,19 +275,39 @@ def evaluate_catalog(cases: Sequence[PairCase]) -> PairEvalResult: }, signals=tuple(build_pair_signals(outcomes)), scorers=scorers, + per_profile={ + profile: _metrics(tuple(item for item in outcomes if item.provenance == profile)) + for profile in sorted({item.provenance for item in outcomes}) + }, + per_mechanism={mechanism: _metrics(tuple(items)) for mechanism, items in sorted(per_mechanism.items())}, + achievable={ + slice_name: _metrics(tuple(item for item in achievable_outcomes if item.slice == slice_name)) + for slice_name in sorted({item.slice for item in achievable_outcomes}) + }, + known_limit=tuple(item.name for item in outcomes if item.known_limit is not None), + loss=_loss(outcomes), + degradations=tuple(degradations), ) def build_pair_signals(outcomes: Sequence[PairOutcome]) -> list[dict[str, object]]: - """Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name.""" + """Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name, profile, and mechanisms.""" signals: list[dict[str, object]] = [] for outcome in outcomes: + if outcome.known_limit is not None: + continue # unachievable by construction; the loop must not chase it + common: dict[str, object] = { + "pair": outcome.name, + "profile": outcome.provenance, + "mechanisms": list(outcome.mechanisms), + "split": outcome.split, + } for miss in outcome.misses: rule_id, path, cwe = (miss.split(":", 2) + ["", ""])[:3] - signals.append({"rule_id": rule_id if rule_id != "-" else "", "signal": "miss", "cwe": cwe, "path": path, "pair": outcome.name}) + signals.append({"rule_id": rule_id if rule_id != "-" else "", "signal": "miss", "cwe": cwe, "path": path, **common}) for leak in outcome.leaks: rule_id = leak.split(":", 1)[0] - signals.append({"rule_id": rule_id, "signal": "fp", "path": leak, "pair": outcome.name, "side": "fixed"}) + signals.append({"rule_id": rule_id, "signal": "fp", "path": leak, "side": "fixed", **common}) return sorted(signals, key=lambda item: (str(item.get("signal")), str(item.get("rule_id")), str(item.get("pair")))) @@ -194,6 +317,18 @@ def select_slice(cases: Sequence[PairCase], slice_name: str | None) -> tuple[Pai return tuple(case for case in cases if case.slice == slice_name) +def select_profile(cases: Sequence[PairCase], profile: str | None) -> tuple[PairCase, ...]: + if not profile or profile == "all": + return tuple(cases) + return tuple(case for case in cases if case.provenance == profile) + + +def select_split(cases: Sequence[PairCase], split: str | None) -> tuple[PairCase, ...]: + if not split or split == "all": + return tuple(cases) + return tuple(case for case in cases if case.split == split) + + def result_payload(result: PairEvalResult) -> dict[str, object]: return { "overall": asdict(result.overall), @@ -201,16 +336,27 @@ def result_payload(result: PairEvalResult) -> dict[str, object]: "outcomes": [asdict(outcome) for outcome in result.outcomes], "signals": list(result.signals), "scorers": { - slice_name: {scorer: asdict(metrics) for scorer, metrics in inner.items()} - for slice_name, inner in result.scorers.items() + slice_name: {scorer: asdict(metrics) for scorer, metrics in inner.items()} for slice_name, inner in result.scorers.items() }, + "per_profile": {name: asdict(metrics) for name, metrics in result.per_profile.items()}, + "per_mechanism": {name: asdict(metrics) for name, metrics in result.per_mechanism.items()}, + "achievable": {name: asdict(metrics) for name, metrics in result.achievable.items()}, + "known_limit": list(result.known_limit), + "loss": {name: dict(counts) for name, counts in result.loss.items()}, + "degradations": list(result.degradations), } -def _expected_for(item: dict[str, object], root: Path) -> tuple[ExpectedFinding, ...]: +# --- catalog helpers ------------------------------------------------------- + + +def _expected_for(item: dict[str, object], root: Path, name: str, mechanisms: frozenset[str]) -> tuple[ExpectedFinding, ...]: inline = item.get("expected") if isinstance(inline, list) and inline: - return tuple(_parse_expected(entry) for entry in inline if isinstance(entry, dict)) + rows = tuple(_parse_expected(entry, name, mechanisms) for entry in inline if isinstance(entry, dict)) + for row in rows: + _validate_expected(row, name) + return rows expected_from = item.get("expected_from") if not expected_from: return () @@ -218,11 +364,24 @@ def _expected_for(item: dict[str, object], root: Path) -> tuple[ExpectedFinding, relpath = str(item["relpath"]) # Pair eval measures ruled sinks. Unruled planted misses (split-sink, CWE-190) # stay on the stage-1/2 fixtures and must not drag local pair_correct below 1.0. - return tuple(entry for entry in manifest.expected if entry.rule_id and (relpath in entry.path or entry.path in relpath)) + derived: list[ExpectedFinding] = [] + for entry in manifest.expected: + if not (entry.rule_id and (relpath in entry.path or entry.path in relpath)): + continue + mechanism = entry.mechanism or _MANIFEST_MECHANISM + if mechanism not in mechanisms: + raise CatalogError(f"pair {name}: unknown mechanism {mechanism!r} in {expected_from}") + derived.append(ExpectedFinding(**{**asdict(entry), "mechanism": mechanism})) + return tuple(derived) -def _parse_expected(item: dict[str, object]) -> ExpectedFinding: +def _parse_expected(item: dict[str, object], name: str, mechanisms: frozenset[str]) -> ExpectedFinding: line_value = item.get("line") + mechanism = item.get("mechanism") + if not isinstance(mechanism, str) or not mechanism: + raise CatalogError(f"pair {name}: expected row is missing mechanism") + if mechanism not in mechanisms: + raise CatalogError(f"pair {name}: unknown mechanism {mechanism!r}; see benchmarks/pairs/mechanisms.toml") return ExpectedFinding( cwe=str(item["cwe"]), vulnerability_class=str(item.get("class", item.get("vulnerability_class", "unknown"))), @@ -232,10 +391,21 @@ def _parse_expected(item: dict[str, object]) -> ExpectedFinding: line=line_value if isinstance(line_value, int) else None, function=str(item["function"]) if "function" in item else None, sink=str(item["sink"]) if "sink" in item else None, + mechanism=mechanism, ) -def _manifest(case: PairCase) -> BenchmarkManifest: +def _validate_expected(row: ExpectedFinding, name: str) -> None: + if (row.sink or "").lower() == "unknown" and not row.function and not row.rule_id: + raise CatalogError(f"pair {name}: expected row has sink=unknown and neither function nor rule_id; label it or drop it") + + +def is_weak_label(row: ExpectedFinding) -> bool: + """A row that can only match by CWE string. Never counts as a detection.""" + return not row.function and not row.rule_id and (not row.sink or row.sink.lower() == "unknown") + + +def _manifest(case: PairCase, expected: Sequence[ExpectedFinding] | None = None) -> BenchmarkManifest: return BenchmarkManifest( name=case.name, language=case.language, @@ -243,7 +413,7 @@ def _manifest(case: PairCase) -> BenchmarkManifest: setup=[], source=BenchmarkSource(type="local", path=str(case.vuln_file)), modes=["quick"], - expected=list(case.expected), + expected=list(case.expected if expected is None else expected), known_noise=[], baselines=[], ) @@ -261,31 +431,127 @@ def _materialize(dest: Path, source: Path, relpath: str) -> Path: return dest -def _quick_scan(target: Path) -> list[StaticFinding]: - _, targets = preprocess_repository(target) - targets = attach_reachability_hints(targets, analyze_entry_points(target, targets)) - return quick_scan_findings(target, targets, rank_targets(targets)) +# --- scans ----------------------------------------------------------------- -def _overlay_scan(target: Path) -> tuple[list[StaticFinding], list[OverlayRecord]]: - _, targets = preprocess_repository(target) - targets = attach_reachability_hints(targets, analyze_entry_points(target, targets)) - findings = quick_scan_findings(target, targets, rank_targets(targets)) - return findings, adjudicate(root=target, targets=targets, findings=findings) +def _targets(root: Path) -> list[FileTarget]: + _, targets = preprocess_repository(root) + return attach_reachability_hints(targets, analyze_entry_points(root, targets)) -def _inventory_only(case: PairCase) -> PairOutcome: +def _quick_scan(target: Path, ruleset: tuple[PatternRule, ...] | None = None) -> list[StaticFinding]: + targets = _targets(target) + findings = quick_scan_findings(target, targets, rank_targets(targets), ruleset) + return [finding for finding in findings if finding.status != "shadow"] + + +@dataclass(frozen=True) +class _Side: + findings: list[StaticFinding] + records: list[OverlayRecord] + parse_failed: int + ranges: dict[str, tuple[tuple[str, int, int], ...]] + + +def _overlay_scan(target: Path, ruleset: tuple[PatternRule, ...] | None = None) -> _Side: + targets = _targets(target) + findings = [f for f in quick_scan_findings(target, targets, rank_targets(targets), ruleset) if f.status != "shadow"] + records = adjudicate(root=target, targets=targets, findings=findings) + parse_failed = 0 + ranges: dict[str, tuple[tuple[str, int, int], ...]] = {} + for item in targets: + try: + text = (target / item.path).read_text(errors="ignore") + except OSError: + parse_failed += 1 + continue + named = named_function_ranges(item.path, text, item.language) + if named is None: + parse_failed += 1 + else: + ranges[item.path] = named + return _Side(findings=findings, records=records, parse_failed=parse_failed, ranges=ranges) + + +def _inventory_only(case: PairCase, ruleset: tuple[PatternRule, ...] | None = None) -> PairOutcome: with tempfile.TemporaryDirectory(prefix="ousast-pair-inv-") as scratch: vuln_root = _materialize(Path(scratch) / "vuln", case.vuln_file, case.relpath) fix_root = _materialize(Path(scratch) / "fixed", case.fixed_file, case.relpath) - return _evaluate_inventory_pair(case, vuln_root, fix_root) + return _evaluate_inventory_pair(case, vuln_root, fix_root, ruleset=ruleset) -def _evaluate_inventory_pair(case: PairCase, vuln_root: Path, fix_root: Path) -> PairOutcome: - vuln_findings = _quick_scan(vuln_root) - fix_findings = _quick_scan(fix_root) +def _evaluate_inventory_pair( + case: PairCase, + vuln_root: Path, + fix_root: Path, + *, + extra: Mapping[str, int] | None = None, + ruleset: tuple[PatternRule, ...] | None = None, +) -> PairOutcome: + vuln_findings = _with_functions(vuln_root, _quick_scan(vuln_root, ruleset)) + fix_findings = _with_functions(fix_root, _quick_scan(fix_root, ruleset)) + return _inventory_outcome(case, vuln_findings, fix_findings, extra=extra) + + +def _with_functions(root: Path, findings: list[StaticFinding]) -> list[StaticFinding]: + """Give regex findings an enclosing function from a parse so function-labeled rows can match (Req 1.2).""" + if not any(finding.function_name is None for finding in findings): + return findings + ranges: dict[str, tuple[tuple[str, int, int], ...]] = {} + for target in _targets(root): + try: + named = named_function_ranges(target.path, (root / target.path).read_text(errors="ignore"), target.language) + except OSError: + continue + if named is not None: + ranges[target.path] = named + out: list[StaticFinding] = [] + for finding in findings: + if finding.function_name is None: + function = function_at(ranges.get(finding.path, ()), finding.line) + if function is not None: + finding = replace(finding, function_name=function) + out.append(finding) + return out + + +def make_hunter_scan(client: object, model: str, *, max_steps: int | None = None) -> HunterScan: + """Run the tool hunter over every file of an isolated pair tree (Req 1.7).""" + from .complexity.map import Hotspot + from .tool_hunter import DEFAULT_MAX_STEPS, run_tool_hunter + + def scan(root: Path) -> list[StaticFinding]: + hotspots = [ + Hotspot( + path=target.path, + function_name=None, + score=1.0, + band="high", + signals={}, + rationale="pair corpus function", + test_hint=None, + inventory_finding_ids=(), + ) + for target in _targets(root) + ] + return _with_functions(root, run_tool_hunter(root, hotspots, client=client, model=model, max_steps=max_steps or DEFAULT_MAX_STEPS)) # type: ignore[arg-type] + + return scan + + +def _inventory_outcome( + case: PairCase, + vuln_findings: list[StaticFinding], + fix_findings: list[StaticFinding], + *, + extra: Mapping[str, int] | None = None, +) -> PairOutcome: + # Weak (CWE-only) rows never match, on this path either (Req 1.3): score the strong rows + # with the benchmark matcher and count the weak rows as misses. + strong = tuple(row for row in case.expected if not is_weak_label(row)) + weak = tuple(row for row in case.expected if is_weak_label(row)) vuln_result = evaluate_benchmark( - run=BenchmarkRun(benchmark_run_id="pair", root=Path("/tmp/pair"), manifest=_manifest(case)), + run=BenchmarkRun(benchmark_run_id="pair", root=Path("/tmp/pair"), manifest=_manifest(case, strong)), mode="quick", findings=vuln_findings, scan_id=None, @@ -297,8 +563,11 @@ def _evaluate_inventory_pair(case: PairCase, vuln_root: Path, fix_root: Path) -> leaks = _fix_leaks(case, fix_findings) detected = recall + 1e-12 >= case.min_recall silent = not leaks - misses = tuple(f"{miss.rule_id or '-'}:{miss.path}:{miss.cwe}" for miss in vuln_result.misses) + misses = tuple(f"{miss.rule_id or '-'}:{miss.path}:{miss.cwe}" for miss in vuln_result.misses) + tuple( + f"{row.rule_id or '-'}:{row.path}:{row.cwe}" for row in weak + ) leak_ids = tuple(finding.finding_id for finding in leaks) + counters = dict(extra or {}) return PairOutcome( name=case.name, slice=case.slice, @@ -317,61 +586,47 @@ def _evaluate_inventory_pair(case: PairCase, vuln_root: Path, fix_root: Path) -> leaks=leak_ids, commit_url=case.commit_url, cve=case.cve, + provenance=case.provenance, + mechanisms=_mechanisms(case), + known_limit=case.known_limit, + split=case.split, + weak_labels=len(weak), + parse_failed_vuln=int(counters.get("parse_failed_vuln", 0)), + parse_failed_fixed=int(counters.get("parse_failed_fixed", 0)), + unadjudicated_vuln=int(counters.get("unadjudicated_vuln", 0)), + unadjudicated_fixed=int(counters.get("unadjudicated_fixed", 0)), + unresolved_labels=int(counters.get("unresolved_labels", 0)), + detection_kinds=("inventory",) if vuln_findings else (), ) -def _evaluate_overlay_pair(case: PairCase, vuln_root: Path, fix_root: Path) -> PairOutcome: - vuln_findings, vuln_overlay = _overlay_scan(vuln_root) - fix_findings, fix_overlay = _overlay_scan(fix_root) - if not _adjudicated(vuln_overlay) and not _adjudicated(fix_overlay): +def _evaluate_overlay_pair(case: PairCase, vuln_root: Path, fix_root: Path, ruleset: tuple[PatternRule, ...] | None = None) -> PairOutcome: + vuln = _overlay_scan(vuln_root, ruleset) + fixed = _overlay_scan(fix_root, ruleset) + loss: dict[str, int] = { + "parse_failed_vuln": vuln.parse_failed, + "parse_failed_fixed": fixed.parse_failed, + "unadjudicated_vuln": sum(1 for record in vuln.records if record.disposition == "unadjudicated"), + "unadjudicated_fixed": sum(1 for record in fixed.records if record.disposition == "unadjudicated"), + } + if not _adjudicated(vuln.records) and not _adjudicated(fixed.records): # Parser/facts could not adjudicate this language; score inventory so # labeled calibration does not collapse when tree-sitter grammars are absent. - vuln_result = evaluate_benchmark( - run=BenchmarkRun(benchmark_run_id="pair", root=Path("/tmp/pair"), manifest=_manifest(case)), - mode="quick", - findings=vuln_findings, - scan_id=None, - scan_run_dir=None, - ) - expected_total = len(case.expected) - matched = vuln_result.metrics.matched_findings_total - recall = matched / expected_total if expected_total else 1.0 - inventory_leaks = _fix_leaks(case, fix_findings) - detected = recall + 1e-12 >= case.min_recall - silent = not inventory_leaks - misses = tuple(f"{miss.rule_id or '-'}:{miss.path}:{miss.cwe}" for miss in vuln_result.misses) - leak_ids = tuple(finding.finding_id for finding in inventory_leaks) - return PairOutcome( - name=case.name, - slice=case.slice, - language=case.language, - origin=case.origin, - detected_vuln=detected, - silent_fix=silent, - pair_correct=detected and silent, - vuln_matched=matched, - vuln_expected=expected_total, - vuln_findings=len(vuln_findings), - fix_findings=len(fix_findings), - fix_leaks=len(inventory_leaks), - recall=recall, - misses=misses, - leaks=leak_ids, - commit_url=case.commit_url, - cve=case.cve, - ) - promoted_vuln = [record for record in vuln_overlay if record.disposition == "promote"] - promoted_fix = [record for record in fix_overlay if record.disposition == "promote"] + loss["unresolved_labels"] = sum(1 for row in case.expected if _label_unresolved(row, vuln.ranges)) + return _inventory_outcome(case, _with_functions(vuln_root, vuln.findings), _with_functions(fix_root, fixed.findings), extra=loss) + detected_vuln = _detections(vuln.records) + detected_fix = _detections(fixed.records) expected_total = len(case.expected) - matched = _overlay_matches(case, promoted_vuln) + matched = sum(1 for row in case.expected if _overlay_matches_expected(row, detected_vuln, case.language, vuln.ranges)) recall = matched / expected_total if expected_total else 1.0 detected = recall + 1e-12 >= case.min_recall - overlay_leaks = _overlay_leaks(case, promoted_fix) + overlay_leaks = _overlay_leaks(case, detected_fix, fixed.ranges) silent = not overlay_leaks + unresolved = sum(1 for row in case.expected if _label_unresolved(row, vuln.ranges)) misses = tuple( - f"{expected.rule_id or '-'}:{expected.path}:{expected.cwe}" - for expected in case.expected - if not _overlay_matches_expected(expected, promoted_vuln) + f"{row.rule_id or '-'}:{row.path}:{row.cwe}" + ("!unresolved" if _label_unresolved(row, vuln.ranges) else "") + for row in case.expected + if not _overlay_matches_expected(row, detected_vuln, case.language, vuln.ranges) ) return PairOutcome( name=case.name, @@ -383,44 +638,174 @@ def _evaluate_overlay_pair(case: PairCase, vuln_root: Path, fix_root: Path) -> P pair_correct=detected and silent, vuln_matched=matched, vuln_expected=expected_total, - vuln_findings=len(promoted_vuln), - fix_findings=len(promoted_fix), + vuln_findings=len(detected_vuln), + fix_findings=len(detected_fix), fix_leaks=len(overlay_leaks), recall=recall, misses=misses, leaks=overlay_leaks, commit_url=case.commit_url, cve=case.cve, + provenance=case.provenance, + mechanisms=_mechanisms(case), + known_limit=case.known_limit, + split=case.split, + weak_labels=sum(1 for row in case.expected if is_weak_label(row)), + detection_kinds=tuple(sorted({record.disposition for record in detected_vuln})), + parse_failed_vuln=loss["parse_failed_vuln"], + parse_failed_fixed=loss["parse_failed_fixed"], + unadjudicated_vuln=loss["unadjudicated_vuln"], + unadjudicated_fixed=loss["unadjudicated_fixed"], + unresolved_labels=unresolved, ) +def _evaluate_hunter_pair(case: PairCase, vuln_root: Path, fix_root: Path, hunter: HunterScan) -> PairOutcome: + """Score the LLM hunter with the same detection and leak rules (Req 1.7).""" + vuln_findings = hunter(vuln_root) + fix_findings = hunter(fix_root) + vuln_ranges = _overlay_scan(vuln_root).ranges + fix_ranges = _overlay_scan(fix_root).ranges + expected_total = len(case.expected) + matched = sum(1 for row in case.expected if any(_hunter_matches_expected(row, finding, vuln_ranges) for finding in vuln_findings)) + recall = matched / expected_total if expected_total else 1.0 + detected = recall + 1e-12 >= case.min_recall + if case.fix_policy == "silent": + leaks = tuple(finding.finding_id for finding in fix_findings) + else: + leaks = tuple( + finding.finding_id + for finding in fix_findings + if any(_hunter_matches_expected(row, finding, fix_ranges) for row in case.expected) + ) + misses = tuple( + f"{row.rule_id or '-'}:{row.path}:{row.cwe}" + for row in case.expected + if not any(_hunter_matches_expected(row, finding, vuln_ranges) for finding in vuln_findings) + ) + return PairOutcome( + name=case.name, + slice=case.slice, + language=case.language, + origin=case.origin, + detected_vuln=detected, + silent_fix=not leaks, + pair_correct=detected and not leaks, + vuln_matched=matched, + vuln_expected=expected_total, + vuln_findings=len(vuln_findings), + fix_findings=len(fix_findings), + fix_leaks=len(leaks), + recall=recall, + misses=misses, + leaks=leaks, + commit_url=case.commit_url, + cve=case.cve, + provenance=case.provenance, + mechanisms=_mechanisms(case), + known_limit=case.known_limit, + split=case.split, + weak_labels=sum(1 for row in case.expected if is_weak_label(row)), + detection_kinds=("hunter",) if vuln_findings else (), + ) + + +# --- matching --------------------------------------------------------------- + + def _adjudicated(records: Sequence[OverlayRecord]) -> bool: return any(record.disposition in {"promote", "demote", "coverage"} for record in records) -def _overlay_matches(case: PairCase, promoted: Sequence[OverlayRecord]) -> int: - return sum(1 for expected in case.expected if _overlay_matches_expected(expected, promoted)) +def _detections(records: Sequence[OverlayRecord]) -> list[OverlayRecord]: + """Promotions plus coverage records that carry a source-to-sink flow (Req 1.1).""" + return [ + record + for record in records + if record.disposition == "promote" or (record.disposition == "coverage" and record.sources and record.sinks) + ] -def _overlay_matches_expected(expected: ExpectedFinding, promoted: Sequence[OverlayRecord]) -> bool: - for record in promoted: +@lru_cache(maxsize=1) +def _sink_ids_by_rule() -> dict[tuple[str, str], frozenset[str]]: + """(language, rule_id) -> fact sink ids that declare that rule (facts are the alias table).""" + try: + facts = load_facts() + except FactLoadError: + return {} + table: dict[tuple[str, str], set[str]] = {} + for sink in facts.sinks: + for rule_id in sink.rule_ids: + table.setdefault((sink.language, rule_id), set()).add(sink.id) + return {key: frozenset(value) for key, value in table.items()} + + +def _fact_language(language: str) -> str: + return {"c_cpp": "c", "cpp": "c", "typescript": "javascript"}.get(language, language) + + +def _in_named_function(record: OverlayRecord, function: str, ranges: FunctionRanges | None) -> bool: + """Design §Scorer rule (1): the record lies inside the labeled function's line range. No range, no match (never name equality).""" + spans = spans_named((ranges or {}).get(record.path, ()), function) + return record.line is not None and any(start <= record.line <= end for start, end in spans) + + +def _label_unresolved(expected: ExpectedFinding, ranges: FunctionRanges | None) -> bool: + """A labeled function that no parsed range of the vulnerable side names cannot be matched; count it as loss.""" + if not expected.function: + return False + return not any(spans_named(spans, expected.function) for spans in (ranges or {}).values()) + + +def _overlay_matches_expected( + expected: ExpectedFinding, detected: Sequence[OverlayRecord], language: str, ranges: FunctionRanges | None = None +) -> bool: + if is_weak_label(expected): + return False + alias_sinks = _sink_ids_by_rule().get((_fact_language(language), expected.rule_id or ""), frozenset()) + for record in detected: + function_ok = True + if expected.function: + function_ok = _in_named_function(record, expected.function, ranges) + if not function_ok: + continue if expected.rule_id and expected.rule_id in record.proposal_id: return True - if expected.cwe and expected.cwe == record.cwe: + if alias_sinks and any(sink in alias_sinks for sink in record.sinks): return True - if expected.sink and expected.sink in record.sinks: + if expected.sink and expected.sink.lower() != "unknown" and expected.sink in record.sinks: + return True + # CWE equality only counts inside a named function (Req 1.2, 1.3). + if expected.function and function_ok and expected.cwe and expected.cwe == record.cwe: return True return False -def _overlay_leaks(case: PairCase, promoted: Sequence[OverlayRecord]) -> tuple[str, ...]: - leaked: list[str] = [] - for record in promoted: - if any(_overlay_matches_expected(expected, [record]) for expected in case.expected) or case.fix_policy == "silent": - leaked.append(record.proposal_id) +def _hunter_matches_expected( + expected: ExpectedFinding, finding: StaticFinding, ranges: dict[str, tuple[tuple[str, int, int], ...]] +) -> bool: + if is_weak_label(expected): + return False + text = f"{finding.title} {finding.rationale} {' '.join(finding.tags)}".lower() + if expected.function: + spans = spans_named(ranges.get(finding.path, ()), expected.function) + if finding.line is None or not any(start <= finding.line <= end for start, end in spans): + return False + if expected.cwe and expected.cwe.lower() in text: + return True + if expected.sink and expected.sink.lower() != "unknown" and expected.sink.lower() in text: + return True + return bool(expected.rule_id and expected.rule_id in finding.finding_id) + + +def _overlay_leaks(case: PairCase, detected_fix: Sequence[OverlayRecord], ranges: FunctionRanges | None = None) -> tuple[str, ...]: if case.fix_policy == "silent": - return tuple(record.proposal_id for record in promoted) - return tuple(leaked) + return tuple(record.proposal_id for record in detected_fix) + return tuple( + record.proposal_id + for record in detected_fix + if any(_overlay_matches_expected(row, [record], case.language, ranges) for row in case.expected) + ) def _fix_leaks(case: PairCase, findings: list[StaticFinding]) -> tuple[StaticFinding, ...]: @@ -438,6 +823,13 @@ def _fix_leaks(case: PairCase, findings: list[StaticFinding]) -> tuple[StaticFin return tuple(leaked) +def _mechanisms(case: PairCase) -> tuple[str, ...]: + return tuple(sorted({row.mechanism for row in case.expected if row.mechanism})) + + +# --- metrics --------------------------------------------------------------- + + def _metrics(outcomes: Sequence[PairOutcome]) -> PairCorpusMetrics: pairs = len(outcomes) detected = sum(1 for item in outcomes if item.detected_vuln) @@ -460,3 +852,18 @@ def _metrics(outcomes: Sequence[PairOutcome]) -> PairCorpusMetrics: labeled_recall=labeled_matched / labeled_expected if labeled_expected else 1.0, youden=(detected / pairs if pairs else 1.0) - (1.0 - (silent / pairs if pairs else 1.0)), ) + + +def _loss(outcomes: Sequence[PairOutcome]) -> dict[str, dict[str, int]]: + loss: dict[str, dict[str, int]] = {} + for item in outcomes: + bucket = loss.setdefault( + item.slice, + {"parse_failed_files": 0, "unadjudicated_proposals": 0, "weak_labels": 0, "known_limit": 0, "unresolved_function_labels": 0}, + ) + bucket["parse_failed_files"] += item.parse_failed_vuln + item.parse_failed_fixed + bucket["unadjudicated_proposals"] += item.unadjudicated_vuln + item.unadjudicated_fixed + bucket["weak_labels"] += item.weak_labels + bucket["unresolved_function_labels"] += item.unresolved_labels + bucket["known_limit"] += 1 if item.known_limit else 0 + return loss diff --git a/src/openultrasast/semantic/overlay.py b/src/openultrasast/semantic/overlay.py index 9fb76c3..703466d 100644 --- a/src/openultrasast/semantic/overlay.py +++ b/src/openultrasast/semantic/overlay.py @@ -12,11 +12,15 @@ from ..policy import load_policy, resolve_severity from ..preprocess import FileTarget from .engines import joern_available, joern_flow from .facts import FactLoadError, SemanticFacts, load_facts -from .taint import Demotion, FileFlow, TaintPath, UnadjudicatedFlow, analyze_file, merge_joern_paths +from .functions import FunctionRange, function_at, named_ranges_from_ir +from .ir import parse_file +from .taint import Demotion, FileFlow, TaintPath, UnadjudicatedFlow, analyze_ir, merge_joern_paths DISPOSITIONS = frozenset({"promote", "demote", "unadjudicated", "coverage"}) UNADJUDICATED_REASONS = frozenset({"parse_failed", "language_unsupported", "flow_incomplete", "facts_unavailable"}) ALLOWED_EVIDENCE = frozenset({"static_corroboration", "suspicion"}) +__all__ = ["OverlayRecord", "adjudicate", "function_at", "FunctionRange", "write_overlay", "finding_from_coverage"] + FORBIDDEN_EVIDENCE = frozenset({"crash_reproduced", "exploit_demonstrated", "patch_validated", "root_cause_explained"}) @@ -40,6 +44,7 @@ class OverlayRecord: origin: str = "inventory" engine: str = "none" language: str = "" + function: str | None = None # enclosing function when the file parsed (additive; pair-corpus-honesty) def __post_init__(self) -> None: if self.disposition not in DISPOSITIONS: @@ -76,13 +81,13 @@ def adjudicate( ) for finding in findings ] - flows = _flows_for(root, targets, loaded, joern_extra) + flows, ranges = _flows_for(root, targets, loaded, joern_extra) records: list[OverlayRecord] = [] claimed: set[tuple[str, int | None, str]] = set() for finding in findings: language = language_by_path.get(finding.path, "") flow = flows.get(finding.path) - record = _record_for_finding(finding, language, flow) + record = _record_for_finding(finding, language, flow, function_at(ranges.get(finding.path, ()), finding.line)) records.append(record) claimed.add((finding.path, finding.line, _sink_hint(finding))) for path, flow in flows.items(): @@ -110,6 +115,7 @@ def adjudicate( origin="overlay", engine=flow.engine, language=language, + function=function_at(ranges.get(path, ()), taint.sink_line), ) ) claimed.add(key) @@ -156,11 +162,17 @@ def _flows_for( targets: Sequence[FileTarget], facts: SemanticFacts, joern_extra: Sequence[TaintPath] | None, -) -> dict[str, FileFlow | UnadjudicatedFlow]: +) -> tuple[dict[str, FileFlow | UnadjudicatedFlow], dict[str, tuple[FunctionRange, ...]]]: flows: dict[str, FileFlow | UnadjudicatedFlow] = {} + ranges: dict[str, tuple[FunctionRange, ...]] = {} for target in targets: text = _read_text(root / target.path) - analyzed = analyze_file(target.path, text, target.language, facts) + ir = parse_file(target.path, text, target.language) + if not ir.parse_ok: + flows[target.path] = UnadjudicatedFlow(path=target.path, reason=ir.reason or "parse_failed", engine=ir.engine) + continue + ranges[target.path] = named_ranges_from_ir(ir, text) + analyzed: FileFlow | UnadjudicatedFlow = analyze_ir(ir, facts.for_language(target.language)) if isinstance(analyzed, FileFlow): extra = tuple(joern_extra) if joern_extra is not None else _joern_paths(target.path, text, target.language) if extra: @@ -173,7 +185,7 @@ def _flows_for( incomplete=analyzed.incomplete, ) flows[target.path] = analyzed - return flows + return flows, ranges def _joern_paths(path: str, text: str, language: str) -> tuple[TaintPath, ...]: @@ -187,7 +199,12 @@ def _joern_paths(path: str, text: str, language: str) -> tuple[TaintPath, ...]: return tuple(paths) -def _record_for_finding(finding: StaticFinding, language: str, flow: FileFlow | UnadjudicatedFlow | None) -> OverlayRecord: +def _record_for_finding( + finding: StaticFinding, + language: str, + flow: FileFlow | UnadjudicatedFlow | None, + function: str | None = None, +) -> OverlayRecord: if flow is None: return _unadjudicated(finding, language, "flow_incomplete", "none") if isinstance(flow, UnadjudicatedFlow): @@ -210,6 +227,7 @@ def _record_for_finding(finding: StaticFinding, language: str, flow: FileFlow | dominating_fact=demotion.dominating_fact, engine=flow.engine, language=language, + function=function, ) if taint is not None and (demotion is None or not demotion.sanitizers): # Sanitizer on the same path would have produced a demotion with sanitizers. @@ -226,6 +244,7 @@ def _record_for_finding(finding: StaticFinding, language: str, flow: FileFlow | evidence_level="static_corroboration", engine=flow.engine, language=language, + function=function, ) if demotion is not None: return OverlayRecord( @@ -242,8 +261,9 @@ def _record_for_finding(finding: StaticFinding, language: str, flow: FileFlow | dominating_fact=demotion.dominating_fact, engine=flow.engine, language=language, + function=function, ) - return _unadjudicated(finding, language, "flow_incomplete", flow.engine) + return _unadjudicated(finding, language, "flow_incomplete", flow.engine, function) def _matching_taint(finding: StaticFinding, flow: FileFlow) -> TaintPath | None: @@ -286,7 +306,7 @@ def _cwe_of(finding: StaticFinding) -> str: return "" -def _unadjudicated(finding: StaticFinding, language: str, reason: str, engine: str) -> OverlayRecord: +def _unadjudicated(finding: StaticFinding, language: str, reason: str, engine: str, function: str | None = None) -> OverlayRecord: return OverlayRecord( proposal_id=finding.finding_id, path=finding.path, @@ -300,6 +320,7 @@ def _unadjudicated(finding: StaticFinding, language: str, reason: str, engine: s evidence_level="static_corroboration", engine=engine, language=language, + function=function, ) diff --git a/tests/test_pair_corpus.py b/tests/test_pair_corpus.py index afd619e..a93856e 100644 --- a/tests/test_pair_corpus.py +++ b/tests/test_pair_corpus.py @@ -3,6 +3,8 @@ import json from pathlib import Path +import pytest + from openultrasast.benchmark import ExpectedFinding from openultrasast.cli import main from openultrasast.gate import LANGUAGE_MANIFESTS @@ -245,3 +247,555 @@ def test_cli_vfc_json_includes_dual_scorers() -> None: assert "overlay" in payload["scorers"]["vfc"] assert "inventory" in payload["scorers"]["vfc"] assert payload["per_slice"]["vfc"]["pairs"] == payload["scorers"]["vfc"]["overlay"]["pairs"] + + +# --- pair-corpus-honesty: honest scorer ------------------------------------- + + +def _case( + tmp_path: Path, name: str, vuln_src: str, fixed_src: str, expected: ExpectedFinding, *, slice_name: str = "sast", **extra: object +) -> PairCase: + vuln = tmp_path / f"{name}-vuln.py" + fixed = tmp_path / f"{name}-fixed.py" + vuln.write_text(vuln_src) + fixed.write_text(fixed_src) + return PairCase( + name=name, + slice=slice_name, + language="python", + origin="test", + vuln_file=vuln, + fixed_file=fixed, + relpath="app.py", + expected=(expected,), + min_recall=1.0, + fix_policy="silent", + provenance=str(extra.get("provenance", "synthetic")), + split=str(extra.get("split", "train")), + known_limit=extra.get("known_limit"), # type: ignore[arg-type] + ) + + +def test_coverage_with_source_counts_as_detection_and_as_leak(tmp_path: Path) -> None: + # hashlib.new is a fact sink the regex inventory does not spell -> overlay coverage, never promote. + src = "import hashlib\nfrom flask import request\n\ndef h():\n data = request.args.get('x')\n return hashlib.new(data)\n" + expected = ExpectedFinding( + cwe="CWE-327", + vulnerability_class="weak hash", + path="app.py", + evidence="", + rule_id="python-weak-hash", + sink="hashlib", + mechanism="config_sink_weak_literal", + ) + both = evaluate_pair(_case(tmp_path, "cov", src, src, expected)) + assert both.detected_vuln and "coverage" in both.detection_kinds + assert not both.silent_fix # the same coverage record on the fixed side is a leak + + +def test_function_scoped_match_rejects_detection_outside_the_named_function(tmp_path: Path) -> None: + src = "from flask import request\n\ndef other():\n return eval(request.args.get('x'))\n\ndef target():\n return 1\n" + row = ExpectedFinding( + cwe="CWE-95", + vulnerability_class="code injection", + path="app.py", + evidence="", + rule_id="python-unsafe-eval", + sink="eval", + function="target", + mechanism="source_reaches_sink", + ) + outcome = evaluate_pair(_case(tmp_path, "fn", src, "def target():\n return 1\n", row)) + assert not outcome.detected_vuln and outcome.silent_fix + row_ok = ExpectedFinding(**{**row.__dict__, "function": "other"}) + assert evaluate_pair(_case(tmp_path, "fn2", src, "def target():\n return 1\n", row_ok)).detected_vuln + + +def test_cwe_only_row_is_a_weak_label_and_never_matches(tmp_path: Path) -> None: + src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n" + weak = ExpectedFinding(cwe="CWE-95", vulnerability_class="code injection", path="app.py", evidence="", mechanism="source_reaches_sink") + outcome = evaluate_pair(_case(tmp_path, "weak", src, "def f():\n return 1\n", weak)) + assert outcome.weak_labels == 1 + assert not outcome.detected_vuln + + +def test_catalog_rejects_unknown_mechanism_and_sink_unknown_without_anchor(tmp_path: Path) -> None: + from openultrasast.pairs import CatalogError, _load_catalog_file + + (tmp_path / "v.py").write_text("x = 1\n") + (tmp_path / "f.py").write_text("x = 1\n") + head = ( + '[[pair]]\nname = "t"\nslice = "sast"\nvuln = "v.py"\nfixed = "f.py"\nrelpath = "app.py"\n\n' + '[[pair.expected]]\ncwe = "CWE-95"\nclass = "x"\npath = "app.py"\n' + ) + (tmp_path / "bad_mech.toml").write_text(head + 'sink = "eval"\nmechanism = "not_a_mechanism"\n') + with pytest.raises(CatalogError, match="unknown mechanism"): + _load_catalog_file(tmp_path / "bad_mech.toml") + (tmp_path / "bad_sink.toml").write_text(head + 'sink = "unknown"\nmechanism = "other"\n') + with pytest.raises(CatalogError, match="sink=unknown"): + _load_catalog_file(tmp_path / "bad_sink.toml") + (tmp_path / "bad_prov.toml").write_text( + head.replace('slice = "sast"\n', 'slice = "sast"\nprovenance = "robot"\n') + 'sink = "eval"\nmechanism = "other"\n' + ) + with pytest.raises(CatalogError, match="unknown provenance"): + _load_catalog_file(tmp_path / "bad_prov.toml") + + +def test_known_limit_pairs_are_evaluated_but_excluded_from_achievable(tmp_path: Path) -> None: + src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n" + row = ExpectedFinding( + cwe="CWE-95", + vulnerability_class="code injection", + path="app.py", + evidence="", + rule_id="python-unsafe-eval", + sink="eval", + mechanism="cross_artifact", + ) + limited = _case(tmp_path, "lim", src, "def f():\n return 1\n", row, known_limit="cross_artifact: needs a properties file") + plain = _case( + tmp_path, + "plain", + src, + "def f():\n return 1\n", + ExpectedFinding(**{**row.__dict__, "mechanism": "source_reaches_sink"}), + provenance="agent", + split="holdout", + ) + result = evaluate_catalog([limited, plain]) + assert result.known_limit == ("lim",) + assert result.per_slice["sast"].pairs == 2 and result.achievable["sast"].pairs == 1 + assert set(result.per_profile) == {"synthetic", "agent"} + assert set(result.per_mechanism) == {"cross_artifact", "source_reaches_sink"} + assert result.loss["sast"]["known_limit"] == 1 + assert "overlay" in result.scorers["sast"] and "inventory" in result.scorers["sast"] + assert result.degradations and result.degradations[0]["reason"] == "hunter_model_unavailable" + + +def test_hunter_scorer_uses_same_rules_with_a_scripted_client(tmp_path: Path) -> None: + from openultrasast.findings import StaticFinding + + src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n" + row = ExpectedFinding( + cwe="CWE-95", + vulnerability_class="code injection", + path="app.py", + evidence="", + sink="eval", + function="f", + mechanism="source_reaches_sink", + ) + + def scripted(root: Path) -> list[StaticFinding]: + text = (root / "app.py").read_text() + if "eval(" not in text: + return [] + return [ + StaticFinding( + finding_id="tool-hunter:app.py:4", + path="app.py", + title="eval of request input", + severity="high", + confidence="medium", + evidence_level="suspicion", + rationale="CWE-95 eval executes attacker input", + line=4, + function_name=None, + reachability_status="unknown", + reachability_evidence=[], + reachability_conditions=[], + tags=[], + ranking_priority=1.0, + ) + ] + + case = _case(tmp_path, "hunt", src, "def f():\n return 1\n", row) + result = evaluate_catalog([case], hunter=scripted) + hunter = result.scorers["sast"]["hunter"] + assert hunter.pair_correct == 1 and hunter.detected_vuln == 1 and hunter.silent_fix == 1 + assert not result.degradations + + +def test_juliet_strcpy_pair_bodies_differ_and_java_hash_is_known_limit() -> None: + cases = {case.name: case for case in select_slice(load_pair_catalog(), "sast")} + strcpy = cases["juliet-c-cwe121-strcpy"] + assert strcpy.vuln_file.read_text() != strcpy.fixed_file.read_text() + assert "dataBadBuffer[10]" in strcpy.vuln_file.read_text() and "data = dataGoodBuffer" in strcpy.fixed_file.read_text() + assert cases["owasp-java-hash"].known_limit and cases["owasp-java-hash"].known_limit.startswith("cross_artifact") + assert all(row.mechanism for case in cases.values() for row in case.expected) + assert all(case.provenance == "synthetic" for case in cases.values()) + + +def test_vfc_rows_name_functions_and_carry_no_weak_labels() -> None: + cases = select_slice(load_pair_catalog(), "vfc") + from openultrasast.pairs import is_weak_label + + assert cases and all(row.function for case in cases for row in case.expected) + assert not any(is_weak_label(row) for case in cases for row in case.expected) + assert {case.split for case in cases} == {"train", "holdout"} + + +def test_cli_pairs_profile_and_split_filters() -> None: + import io + from contextlib import redirect_stdout + + buf = io.StringIO() + with redirect_stdout(buf): + assert main(["pairs", "--slice", "sast", "--profile", "synthetic", "--split", "holdout", "--json"]) == 0 + payload = json.loads(buf.getvalue()) + assert payload["overall"]["pairs"] >= 3 + assert set(payload["per_profile"]) == {"synthetic"} + assert "per_mechanism" in payload and "loss" in payload and "achievable" in payload + assert all(item["split"] == "holdout" for item in payload["outcomes"]) + + +def test_new_slices_load_offline_and_score_with_profiles() -> None: + from openultrasast.pairs import PROVENANCES + + catalog = load_pair_catalog() + # agent-vfc rows enter the loaded catalog only after human review (Req 5.3); candidates wait in a separate file. + candidates = Path("benchmarks/pairs/agent-vfc/catalog-candidates.toml") + assert candidates.is_file() and "reviewer" in candidates.read_text() + assert all(case.provenance == "agent" for case in select_slice(catalog, "agent-vfc")) + for slice_name, expected_provenance in (("vibe-py", "human"), ("vfc-js", "human")): + cases = select_slice(catalog, slice_name) + assert cases, slice_name + assert all(case.vuln_file.is_file() and case.fixed_file.is_file() for case in cases) + assert all(case.license for case in cases), slice_name + assert all(case.provenance == expected_provenance for case in cases), slice_name + assert {case.split for case in cases} <= {"train", "holdout"} + header = cases[0].vuln_file.read_text()[:600] + assert "Provenance:" in header and "license:" in header + result = evaluate_catalog(cases[:2]) + assert result.per_slice[slice_name].pairs == 2 + assert set(result.per_profile) <= PROVENANCES and expected_provenance in result.per_profile + assert result.per_mechanism and slice_name in result.loss + + +# --- review round 1 remediation (RED first) ----------------------------------- + + +def test_weak_label_never_matches_on_inventory_fallback(tmp_path: Path) -> None: + # A syntax error forces parse_failed on both sides -> overlay falls back to inventory. + src = "from flask import request\n\ndef f(:\n return eval(request.args.get('x'))\n" + weak = ExpectedFinding(cwe="CWE-95", vulnerability_class="code injection", path="app.py", evidence="", mechanism="source_reaches_sink") + outcome = evaluate_pair(_case(tmp_path, "weakfb", src, "def f():\n return 1\n", weak)) + assert outcome.parse_failed_vuln == 1 + assert outcome.weak_labels == 1 + assert not outcome.detected_vuln + assert outcome.misses == ("-:app.py:CWE-95",) + + +def test_sink_match_is_exact_or_alias_not_substring() -> None: + from openultrasast.pairs import _overlay_matches_expected + from openultrasast.semantic import OverlayRecord + + record = OverlayRecord( + proposal_id="overlay-coverage:app.py:3:execute", + path="app.py", + line=3, + disposition="coverage", + reason="x", + cwe="CWE-89", + sources=("request",), + sinks=("execute",), + sanitizers=(), + evidence_level="static_corroboration", + ) + exec_row = ExpectedFinding( + cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", sink="exec", mechanism="source_reaches_sink" + ) + assert not _overlay_matches_expected(exec_row, [record], "python") + exact = ExpectedFinding( + cwe="CWE-89", vulnerability_class="x", path="app.py", evidence="", sink="execute", mechanism="source_reaches_sink" + ) + assert _overlay_matches_expected(exact, [record], "python") + alias = ExpectedFinding( + cwe="CWE-89", vulnerability_class="x", path="app.py", evidence="", rule_id="python-sql-injection", mechanism="source_reaches_sink" + ) + assert _overlay_matches_expected(alias, [record], "python") + + +def test_vfc_style_slice_rows_require_a_named_function(tmp_path: Path) -> None: + from openultrasast.pairs import CatalogError, _load_catalog_file + + (tmp_path / "v.js").write_text("x = 1\n") + (tmp_path / "f.js").write_text("x = 1\n") + head = ( + '[[pair]]\nname = "t"\nslice = "vfc-js"\nvuln = "v.js"\nfixed = "f.js"\nrelpath = "a.js"\n\n' + '[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "a.js"\nmechanism = "source_reaches_sink"\n' + ) + (tmp_path / "nofn.toml").write_text(head + 'sink = "exec"\n') + with pytest.raises(CatalogError, match="function"): + _load_catalog_file(tmp_path / "nofn.toml") + (tmp_path / "anon.toml").write_text(head + 'function = ""\n') + with pytest.raises(CatalogError, match="function"): + _load_catalog_file(tmp_path / "anon.toml") + (tmp_path / "ok.toml").write_text(head + 'function = "run"\n') + assert _load_catalog_file(tmp_path / "ok.toml")[0].expected[0].function == "run" + + +def test_known_limit_pairs_emit_no_improve_signals(tmp_path: Path) -> None: + src = "def f():\n return 1\n" + row = ExpectedFinding( + cwe="CWE-95", + vulnerability_class="x", + path="app.py", + evidence="", + rule_id="python-unsafe-eval", + sink="eval", + mechanism="cross_artifact", + ) + limited = _case(tmp_path, "lim2", src, src, row, known_limit="cross_artifact: needs a properties file") + result = evaluate_catalog([limited]) + assert not result.outcomes[0].detected_vuln + assert all(signal["pair"] != "lim2" for signal in result.signals) + + +# --- review round 2 remediation (RED first) ----------------------------------- + + +def test_function_at_skips_anonymous_and_module_ranges() -> None: + from openultrasast.semantic.overlay import function_at + + ranges = (("", 1, 40), ("handler", 3, 20), ("", 5, 12)) + assert function_at(ranges, 7) == "handler" # innermost *named* function, never "" + assert function_at(ranges, 25) is None + assert function_at((("", 1, 9),), 4) is None + + +@pytest.mark.semantic +def test_promotion_inside_a_callback_matches_the_enclosing_named_function(tmp_path: Path) -> None: + from openultrasast.semantic.extra import has_semantic_extra + + if not has_semantic_extra(): + pytest.skip("tree-sitter extra absent") + src = ( + "const cp = require('child_process');\n" + "function handler(req, res) {\n" + " [req.query.cmd].forEach(function (c) {\n" + " cp.exec(c);\n" + " });\n" + "}\n" + ) + fixed = "function handler(req, res) {\n res.end('ok');\n}\n" + vuln = tmp_path / "v.js" + fix = tmp_path / "f.js" + vuln.write_text(src) + fix.write_text(fixed) + row = ExpectedFinding( + cwe="CWE-78", vulnerability_class="x", path="app.js", evidence="", function="handler", sink="exec", mechanism="source_reaches_sink" + ) + case = PairCase( + name="cb", + slice="vfc-js", + language="javascript", + origin="test", + vuln_file=vuln, + fixed_file=fix, + relpath="app.js", + expected=(row,), + min_recall=1.0, + fix_policy="silent", + provenance="human", + ) + outcome = evaluate_pair(case) + assert outcome.detected_vuln, outcome + assert outcome.pair_correct + + +def test_loader_rejects_reserved_words_as_function_labels(tmp_path: Path) -> None: + from openultrasast.pairs import CatalogError, _load_catalog_file + + (tmp_path / "v.js").write_text("x = 1\n") + (tmp_path / "f.js").write_text("x = 1\n") + head = ( + '[[pair]]\nname = "t"\nslice = "vfc-js"\nvuln = "v.js"\nfixed = "f.js"\nrelpath = "a.js"\n\n' + '[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "a.js"\nmechanism = "source_reaches_sink"\nsink = "exec"\n' + ) + for bad in ("function", "def", "return"): + (tmp_path / "bad.toml").write_text(head + f'function = "{bad}"\n') + with pytest.raises(CatalogError, match="function"): + _load_catalog_file(tmp_path / "bad.toml") + + +def test_agent_vfc_stays_empty_until_reviewed() -> None: + catalog = load_pair_catalog() + loaded = select_slice(catalog, "agent-vfc") + candidates = Path("benchmarks/pairs/agent-vfc/catalog-candidates.toml") + # Req 5.3: nothing loads until a human sets `reviewer`; the candidates file must not be empty meanwhile. + assert candidates.is_file() and candidates.read_text().count("[[pair]]") >= 1 + assert loaded == () or all(case.provenance == "agent" for case in loaded) + reviewed_in_recipes = 'reviewer = "pending"' not in Path("benchmarks/pairs/agent-vfc/recipes.toml").read_text() + assert bool(loaded) == reviewed_in_recipes + + +# --- debug round after review 3: shared named ranges, containment-only matching ----- + + +def _js_case(tmp_path: Path, name: str, src: str, function: str) -> PairCase: + vuln = tmp_path / f"{name}-v.js" + fix = tmp_path / f"{name}-f.js" + vuln.write_text(src) + fix.write_text("function untouched() {\n return 1;\n}\n") + row = ExpectedFinding( + cwe="CWE-78", vulnerability_class="x", path="app.js", evidence="", function=function, sink="exec", mechanism="source_reaches_sink" + ) + return PairCase( + name=name, + slice="vfc-js", + language="javascript", + origin="test", + vuln_file=vuln, + fixed_file=fix, + relpath="app.js", + expected=(row,), + min_recall=1.0, + fix_policy="silent", + provenance="human", + ) + + +_JS_FORMS = { + "arrow": "const cp = require('child_process');\nconst h = (req, res) => {\n cp.exec(req.query.cmd);\n};\n", + "exports": "const cp = require('child_process');\nexports.h = function (req, res) {\n cp.exec(req.query.cmd);\n};\n", + "method": "const cp = require('child_process');\nclass Ctl {\n h(req, res) {\n cp.exec(req.query.cmd);\n }\n}\n", + "property": ( + "const cp = require('child_process');\nmodule.exports = {\n h: function (req, res) {\n cp.exec(req.query.cmd);\n },\n};\n" + ), +} + + +@pytest.mark.semantic +@pytest.mark.parametrize("form", sorted(_JS_FORMS)) +def test_declarator_named_functions_match_their_labels(tmp_path: Path, form: str) -> None: + from openultrasast.semantic.extra import has_semantic_extra + + if not has_semantic_extra(): + pytest.skip("tree-sitter extra absent") + outcome = evaluate_pair(_js_case(tmp_path, form, _JS_FORMS[form], "h")) + assert outcome.detected_vuln, (form, outcome) + + +@pytest.mark.semantic +def test_containment_matches_outer_label_and_rejects_inner_label(tmp_path: Path) -> None: + from openultrasast.semantic.extra import has_semantic_extra + + if not has_semantic_extra(): + pytest.skip("tree-sitter extra absent") + src = ( + "const cp = require('child_process');\n" + "function outer(req) {\n" + " function inner(c) {\n" + " cp.exec(c);\n" + " }\n" + " inner(req.query.cmd);\n" + " cp.exec(req.query.top);\n" + "}\n" + ) + # The sink inside `inner` also lies inside `outer`'s range: containment says yes, name equality would say no. + assert evaluate_pair(_js_case(tmp_path, "outer", src, "outer")).detected_vuln + # A label naming `inner` must not match the promote at line 7, which is outside `inner`. + only_top = src.replace(" cp.exec(c);\n", " return c;\n") + assert not evaluate_pair(_js_case(tmp_path, "inner", only_top, "inner")).detected_vuln + + +def test_unresolvable_function_label_is_a_visible_loss_not_a_silent_miss(tmp_path: Path) -> None: + src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n" + row = ExpectedFinding( + cwe="CWE-95", + vulnerability_class="x", + path="app.py", + evidence="", + function="does_not_exist", + sink="eval", + mechanism="source_reaches_sink", + ) + outcome = evaluate_pair(_case(tmp_path, "unres", src, "def f():\n return 1\n", row)) + assert not outcome.detected_vuln + assert outcome.unresolved_labels == 1 + assert any(miss.endswith("!unresolved") for miss in outcome.misses) + result = evaluate_catalog([_case(tmp_path, "unres2", src, "def f():\n return 1\n", row)]) + assert result.loss["sast"]["unresolved_function_labels"] == 1 + + +def test_named_function_ranges_use_declarator_names() -> None: + from openultrasast.semantic.extra import has_semantic_extra + from openultrasast.semantic.functions import declarator_name, named_function_ranges + + assert declarator_name("const listProcessesOnPort = module.exports.listProcessesOnPort = async port => {") == "listProcessesOnPort" + assert declarator_name(" exec('df -k ' + path, function(err, stdout) {") is None + assert declarator_name(" function (req, res) {") is None + if not has_semantic_extra(): + pytest.skip("tree-sitter extra absent") + src = "\n".join( + [ + "function a() { return 1; }", + "const b = (x) => { return x; };", + "exports.c = function (y) { return y; };", + "class K {", + " d(z) {", + " return z;", + " }", + "}", + "const o = {", + " e: function (w) {", + " return w;", + " },", + "};", + "", + ] + ) + ranges = named_function_ranges("f.js", src, "javascript") + assert ranges is not None + assert {name for name, _, _ in ranges} >= {"a", "b", "c", "d", "e"} + assert "" not in {name for name, _, _ in ranges} + + +# --- review round 4 remediation (RED first) ----------------------------------- + + +@pytest.mark.semantic +def test_c_function_with_return_type_on_previous_line_keeps_its_parser_name() -> None: + from openultrasast.semantic.extra import has_semantic_extra + from openultrasast.semantic.functions import function_at, named_function_ranges + + if not has_semantic_extra(): + pytest.skip("tree-sitter extra absent") + ranges = named_function_ranges("a.c", "static int\nfoo(int x)\n{\n return x;\n}\n", "c") + assert ranges is not None and any(name == "foo" and start <= 4 <= end for name, start, end in ranges) + assert function_at(ranges, 4) == "foo" + + +def test_declarator_name_knows_perl_subs_and_never_names_calls() -> None: + from openultrasast.semantic.functions import declarator_name + + assert declarator_name("sub link_hash_cert {") == "link_hash_cert" + assert declarator_name(" return foo(bar)") is None + assert declarator_name(" print(x)") is None + assert declarator_name("synchronized (lock) {") is None + assert declarator_name("int main(int argc, char **argv)") == "main" + + +# --- review round 5 remediation (RED first) ----------------------------------- + + +def test_declarator_name_strips_cpp_qualification_to_match_unqualified_labels() -> None: + from openultrasast.semantic.functions import declarator_name + + assert declarator_name("bool FileReaderLoader::ArrayBufferResult(int a) {") == "ArrayBufferResult" + assert declarator_name("void ns::Klass::run(const std::string& s) const {") == "run" + assert declarator_name("std::string Foo::bar() {") == "bar" + + +@pytest.mark.semantic +def test_cpp_qualified_method_range_is_found_by_its_unqualified_label() -> None: + from openultrasast.semantic.extra import has_semantic_extra + from openultrasast.semantic.functions import function_at, named_function_ranges, spans_named + + if not has_semantic_extra(): + pytest.skip("tree-sitter extra absent") + ranges = named_function_ranges("a.cc", "bool K::m(int a) {\n return a;\n}\n", "cpp") + assert ranges is not None + assert spans_named(ranges, "m") == ((1, 3),) + assert function_at(ranges, 2) == "m"