mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5
1af3db37f6
output_encoding had exactly ONE pair in the whole corpus. It now has 41. 238 pairs were harvested from OWASP BenchmarkJava (2740 generated servlets, GPL-2.0, with a ground-truth CSV) by matching each really-vulnerable test to the safe test of the same category it most resembles -- adjacent tests come from one template, so they differ in the sink or the sanitiser rather than being unrelated programs. Median similarity 0.91, against the 64% of the older `sast` OWASP entries that the group-4 stratification flagged as different programs. Only 40 are vendored. One representative of each of the seven categories was run through the model BEFORE vendoring, and only xss distinguished: its fix adds ESAPI encoding, which the Java sanitizer facts recognise, so the vulnerable side entails and the fixed side drops to corroborated. cmdi, sqli, ldapi and xpathi put the sink in a helper class outside the servlet and resolved no flow at all; pathtraver found the flow but its "safe" twin carries no distinguishing call, so text similarity had matched a twin that is not semantically the fix -- a subtler form of the very defect the harvest was designed to avoid. Those 198 pairs are deliberately left out: adding pairs the model cannot read would inflate the corpus while making every number less interpretable. Evidence for the vendored slice is 3 of 3 distinguished, which is a small sample and is stated as such in the catalog header rather than implied to be more. The slice is marked SYNTHETIC in its header: it measures breadth across a weakness class and must never stand in for real-world performance. The corpus already shows a +16% overfitting gap, and a large generated slice blended into a headline would widen it invisibly. Two things recorded rather than hidden: BenchmarkJava's LICENSE is GPL-2.0 while the older sast entries declare GPL-3.0 for OWASP, which looks like an error and is not copied; and OWASP-Benchmark/BenchmarkJava already straddled the train/holdout split before these pairs (owasp-java-cmdi train, owasp-java-sqli holdout), so the new all-holdout pairs enlarge an existing straddle rather than create one. The two vendored-count guards did their job -- a corpus change could not land silently -- and are updated as the acknowledgement. Hygiene artifact regenerated: 277 -> 317 vendored pairs. 720 passing, mypy/ruff clean, gates byte-identical. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
470 lines
15 KiB
JSON
470 lines
15 KiB
JSON
{
|
|
"command": "python -c 'from openultrasast.pairs import ...; duplicate_groups / split_by_repository over the vendored catalog'",
|
|
"cross_split_duplicates": [
|
|
[
|
|
"curl-cve-2023-38545",
|
|
"curl-cve-2023-38545-do_socks5"
|
|
],
|
|
[
|
|
"etude-lexical-markdown-editorconfig-c61a8b",
|
|
"popmark-editorconfig-e5ac40"
|
|
],
|
|
[
|
|
"openniw-jobs-0fc947",
|
|
"openniw-jobs-b74de3"
|
|
],
|
|
[
|
|
"openssl-cve-2020-1967",
|
|
"openssl-cve-2020-1967-tls1_check_sig_alg"
|
|
],
|
|
[
|
|
"openssl-cve-2020-1971",
|
|
"openssl-cve-2020-1971-general_name_cmp"
|
|
],
|
|
[
|
|
"openssl-cve-2021-3712",
|
|
"openssl-cve-2021-3712-append_ia5"
|
|
],
|
|
[
|
|
"openssl-cve-2022-3602",
|
|
"openssl-cve-2022-3602-ossl_punycode_decode"
|
|
],
|
|
[
|
|
"openssl-cve-2022-3786",
|
|
"openssl-cve-2022-3786-ossl_punycode_decode"
|
|
],
|
|
[
|
|
"openssl-cve-2023-0286",
|
|
"openssl-cve-2023-0286-general_name_cmp"
|
|
]
|
|
],
|
|
"duplicate_groups": [
|
|
[
|
|
"anastasios-pw-session-14ec1a",
|
|
"astroclaw-pw-session-1553a8"
|
|
],
|
|
[
|
|
"curl-cve-2023-38545",
|
|
"curl-cve-2023-38545-do_socks5"
|
|
],
|
|
[
|
|
"etude-lexical-markdown-editorconfig-c61a8b",
|
|
"popmark-editorconfig-e5ac40"
|
|
],
|
|
[
|
|
"openniw-jobs-0fc947",
|
|
"openniw-jobs-b74de3"
|
|
],
|
|
[
|
|
"openssl-cve-2020-1967",
|
|
"openssl-cve-2020-1967-tls1_check_sig_alg"
|
|
],
|
|
[
|
|
"openssl-cve-2020-1971",
|
|
"openssl-cve-2020-1971-general_name_cmp"
|
|
],
|
|
[
|
|
"openssl-cve-2021-3712",
|
|
"openssl-cve-2021-3712-append_ia5"
|
|
],
|
|
[
|
|
"openssl-cve-2022-3602",
|
|
"openssl-cve-2022-3602-ossl_punycode_decode"
|
|
],
|
|
[
|
|
"openssl-cve-2022-3786",
|
|
"openssl-cve-2022-3786-ossl_punycode_decode"
|
|
],
|
|
[
|
|
"openssl-cve-2023-0286",
|
|
"openssl-cve-2023-0286-general_name_cmp"
|
|
]
|
|
],
|
|
"generated": "2026-09-06",
|
|
"identical_twins": [
|
|
"dsvw-dsvw-do-get",
|
|
"python-insecure-app-main-try-hack-me",
|
|
"pythonssti-main-read-root",
|
|
"vampi-books-get-by-title",
|
|
"vulnpy-deserialization-do-pickle-load"
|
|
],
|
|
"known_limit": [
|
|
"owasp-java-hash"
|
|
],
|
|
"note": "Repositories in `straddling` have pairs on both sides of the train/holdout split, so a learning path can be taught by one file of a repository and evaluated on another. Pairs in `duplicate_groups` share a vulnerable excerpt byte for byte after the provenance header is stripped. Both cap what any detector number can mean. Groups in `cross_split_duplicates` sit on both sides of the split while being the same code, so the train row teaches the holdout row exactly; forks of one project defeat the repository-level split.",
|
|
"owasp_note": "The 40 vendored BenchmarkJava xss pairs are all holdout. OWASP-Benchmark/BenchmarkJava already straddled the split before them (owasp-java-cmdi train, owasp-java-sqli holdout), so the new pairs enlarge an existing straddle rather than create one. It matters less than it did -- the learning path a train row could teach is deleted -- but it is recorded, not hidden.",
|
|
"regenerated": "2026-09-08 after the owasp slice was vendored",
|
|
"straddling": [
|
|
[
|
|
"Contrast-Security-OSS/DjanGoat",
|
|
[
|
|
"djangoat-analytics-hits-by-ip",
|
|
"djangoat-views-reset-password",
|
|
"djangoat-views-user-view"
|
|
]
|
|
],
|
|
[
|
|
"Contrast-Security-OSS/vulnpy",
|
|
[
|
|
"vulnpy-cmdi-do-os-system",
|
|
"vulnpy-cmdi-do-subprocess-popen",
|
|
"vulnpy-deserialization-do-pickle-load"
|
|
]
|
|
],
|
|
[
|
|
"LinuxUser255/Vulnerable_Python_Apps",
|
|
[
|
|
"vulnerable-python-apps-bad-mvc-do-get",
|
|
"vulnerable-python-apps-dir-traversal-do-get"
|
|
]
|
|
],
|
|
[
|
|
"OWASP-Benchmark/BenchmarkJava",
|
|
[
|
|
"owasp-java-cmdi",
|
|
"owasp-java-hash",
|
|
"owasp-java-sqli",
|
|
"owasp-xss-benchmarktest00013",
|
|
"owasp-xss-benchmarktest00014",
|
|
"owasp-xss-benchmarktest00030",
|
|
"owasp-xss-benchmarktest00036",
|
|
"owasp-xss-benchmarktest00041",
|
|
"owasp-xss-benchmarktest00144",
|
|
"owasp-xss-benchmarktest00145",
|
|
"owasp-xss-benchmarktest00146",
|
|
"owasp-xss-benchmarktest00148",
|
|
"owasp-xss-benchmarktest00149",
|
|
"owasp-xss-benchmarktest00150",
|
|
"owasp-xss-benchmarktest00152",
|
|
"owasp-xss-benchmarktest00153",
|
|
"owasp-xss-benchmarktest00154",
|
|
"owasp-xss-benchmarktest00155",
|
|
"owasp-xss-benchmarktest00156",
|
|
"owasp-xss-benchmarktest00157",
|
|
"owasp-xss-benchmarktest00276",
|
|
"owasp-xss-benchmarktest00279",
|
|
"owasp-xss-benchmarktest00280",
|
|
"owasp-xss-benchmarktest00284",
|
|
"owasp-xss-benchmarktest00287",
|
|
"owasp-xss-benchmarktest00290",
|
|
"owasp-xss-benchmarktest00291",
|
|
"owasp-xss-benchmarktest00292",
|
|
"owasp-xss-benchmarktest00301",
|
|
"owasp-xss-benchmarktest00375",
|
|
"owasp-xss-benchmarktest00376",
|
|
"owasp-xss-benchmarktest00378",
|
|
"owasp-xss-benchmarktest00380",
|
|
"owasp-xss-benchmarktest00382",
|
|
"owasp-xss-benchmarktest00383",
|
|
"owasp-xss-benchmarktest00384",
|
|
"owasp-xss-benchmarktest00385",
|
|
"owasp-xss-benchmarktest00387",
|
|
"owasp-xss-benchmarktest00388",
|
|
"owasp-xss-benchmarktest00390",
|
|
"owasp-xss-benchmarktest00392",
|
|
"owasp-xss-benchmarktest00395",
|
|
"owasp-xss-benchmarktest00467"
|
|
]
|
|
],
|
|
[
|
|
"OWASP-Benchmark/BenchmarkPython",
|
|
[
|
|
"owasp-python-cmdi",
|
|
"owasp-python-codeinj",
|
|
"owasp-python-deser",
|
|
"owasp-python-hash",
|
|
"owasp-python-sqli"
|
|
]
|
|
],
|
|
[
|
|
"anotherik/ThreatByte",
|
|
[
|
|
"threatbyte-api-v1-delete",
|
|
"threatbyte-api-v1-get",
|
|
"threatbyte-routes-dashboard"
|
|
]
|
|
],
|
|
[
|
|
"anxolerd/dvpwa",
|
|
[
|
|
"dvpwa-app-init",
|
|
"dvpwa-middlewares-session-middleware",
|
|
"dvpwa-student-create"
|
|
]
|
|
],
|
|
[
|
|
"arichardson/juliet-test-suite-c",
|
|
[
|
|
"juliet-c-cwe121-strcpy",
|
|
"juliet-c-cwe134-printf"
|
|
]
|
|
],
|
|
[
|
|
"bigfivefoods/supplieradvisor-mvp",
|
|
[
|
|
"supplieradvisor-mvp-route-387bcf",
|
|
"supplieradvisor-mvp-route-a69e8f"
|
|
]
|
|
],
|
|
[
|
|
"chromium/chromium",
|
|
[
|
|
"chromium-cve-2019-5786",
|
|
"chromium-cve-2022-2162"
|
|
]
|
|
],
|
|
[
|
|
"curl/curl",
|
|
[
|
|
"curl-cve-2020-8177",
|
|
"curl-cve-2020-8231-curl_multi_add_handle",
|
|
"curl-cve-2020-8231-curl_open",
|
|
"curl-cve-2020-8284",
|
|
"curl-cve-2020-8284-config_init",
|
|
"curl-cve-2020-8286",
|
|
"curl-cve-2021-22876",
|
|
"curl-cve-2021-22890",
|
|
"curl-cve-2021-22898",
|
|
"curl-cve-2021-22901-ossl_connect_step1",
|
|
"curl-cve-2021-22924",
|
|
"curl-cve-2021-22924-nss_do_connect",
|
|
"curl-cve-2021-22924-servercert",
|
|
"curl-cve-2021-22925",
|
|
"curl-cve-2021-22945",
|
|
"curl-cve-2021-22946",
|
|
"curl-cve-2021-22946-imap_state_capability_re",
|
|
"curl-cve-2021-22946-pop3_state_capa_resp",
|
|
"curl-cve-2021-22947",
|
|
"curl-cve-2021-22947-imap_state_starttls_resp",
|
|
"curl-cve-2021-22947-pop3_state_starttls_resp",
|
|
"curl-cve-2021-22947-smtp_state_starttls_resp",
|
|
"curl-cve-2022-22576-conn_free",
|
|
"curl-cve-2022-27774",
|
|
"curl-cve-2022-27775",
|
|
"curl-cve-2022-27776",
|
|
"curl-cve-2022-27778",
|
|
"curl-cve-2022-27780",
|
|
"curl-cve-2022-27782-curl_free_primary_ssl_co",
|
|
"curl-cve-2022-27782-curl_init_userdefined",
|
|
"curl-cve-2022-27782-gtls_shutdown",
|
|
"curl-cve-2022-27782-nss_setup_connect",
|
|
"curl-cve-2022-27782-ossl_connect_step1",
|
|
"curl-cve-2022-30115",
|
|
"curl-cve-2022-32206",
|
|
"curl-cve-2022-32207",
|
|
"curl-cve-2022-32208",
|
|
"curl-cve-2023-38545",
|
|
"curl-cve-2023-38545-do_socks5",
|
|
"curl-cve-2026-9080"
|
|
]
|
|
],
|
|
[
|
|
"dayflower/etude-lexical-markdown",
|
|
[
|
|
"etude-lexical-markdown-editorconfig-c61a8b",
|
|
"etude-lexical-markdown-editorconfig-db4f54"
|
|
]
|
|
],
|
|
[
|
|
"dolevf/Damn-Vulnerable-GraphQL-Application",
|
|
[
|
|
"damn-vulnerable-graphql-application-views-mutate",
|
|
"damn-vulnerable-graphql-application-views-resolve-pastes",
|
|
"damn-vulnerable-graphql-application-views-resolve-system-dia"
|
|
]
|
|
],
|
|
[
|
|
"erev0s/VAmPI",
|
|
[
|
|
"vampi-books-get-by-title",
|
|
"vampi-user-model-get-user",
|
|
"vampi-users-update-password"
|
|
]
|
|
],
|
|
[
|
|
"fportantier/vulpy",
|
|
[
|
|
"vulpy-libuser-create",
|
|
"vulpy-libuser-login",
|
|
"vulpy-libuser-password-change"
|
|
]
|
|
],
|
|
[
|
|
"manuelz120/extremely-vulnerable-flask-app",
|
|
[
|
|
"extremely-vulnerable-flask-app-account-before-request",
|
|
"extremely-vulnerable-flask-app-account-search",
|
|
"extremely-vulnerable-flask-app-signup-validate-token"
|
|
]
|
|
],
|
|
[
|
|
"naryal2580/vfapi",
|
|
[
|
|
"vfapi-main-delete-user",
|
|
"vfapi-main-put-user",
|
|
"vfapi-main-sql-return-users-from-username"
|
|
]
|
|
],
|
|
[
|
|
"openssl/openssl",
|
|
[
|
|
"openssl-cve-2014-0160",
|
|
"openssl-cve-2020-1967",
|
|
"openssl-cve-2020-1967-tls1_check_sig_alg",
|
|
"openssl-cve-2020-1971",
|
|
"openssl-cve-2020-1971-general_name_cmp",
|
|
"openssl-cve-2020-25659",
|
|
"openssl-cve-2021-23840",
|
|
"openssl-cve-2021-23840-evp_encryptdecryptupdate",
|
|
"openssl-cve-2021-23841",
|
|
"openssl-cve-2021-3449",
|
|
"openssl-cve-2021-3449-def_generate_session_id",
|
|
"openssl-cve-2021-3449-init_sig_algs",
|
|
"openssl-cve-2021-3449-init_sig_algs_cert",
|
|
"openssl-cve-2021-3449-ssl_ctx_set_alpn_protos",
|
|
"openssl-cve-2021-3449-ssl_generate_master_secr",
|
|
"openssl-cve-2021-3449-tls_construct_ctos_early",
|
|
"openssl-cve-2021-3449-tls_handle_alpn",
|
|
"openssl-cve-2021-3449-tls_parse_ctos_renegotia",
|
|
"openssl-cve-2021-3449-tls_process_certificate_",
|
|
"openssl-cve-2021-3449-tls_process_finished",
|
|
"openssl-cve-2021-3450",
|
|
"openssl-cve-2021-3711",
|
|
"openssl-cve-2021-3711-pkey_sm2_decrypt",
|
|
"openssl-cve-2021-3711-sm2_asym_decrypt",
|
|
"openssl-cve-2021-3712",
|
|
"openssl-cve-2021-3712-append_ia5",
|
|
"openssl-cve-2021-3712-general_name_print",
|
|
"openssl-cve-2021-3712-i2r_pci",
|
|
"openssl-cve-2021-3712-ia5ncasecmp",
|
|
"openssl-cve-2021-3712-netscape_spki_print",
|
|
"openssl-cve-2021-3712-ossl_cmp_hdr_push1_freet",
|
|
"openssl-cve-2021-3712-print_qualifiers",
|
|
"openssl-cve-2021-4044",
|
|
"openssl-cve-2021-4044-tls_post_process_server_",
|
|
"openssl-cve-2022-0778",
|
|
"openssl-cve-2022-1292",
|
|
"openssl-cve-2022-1343",
|
|
"openssl-cve-2022-1434",
|
|
"openssl-cve-2022-2068-hash_dir",
|
|
"openssl-cve-2022-29242",
|
|
"openssl-cve-2022-3358",
|
|
"openssl-cve-2022-3358-evp_cipher_init_internal",
|
|
"openssl-cve-2022-3602",
|
|
"openssl-cve-2022-3602-ossl_punycode_decode",
|
|
"openssl-cve-2022-37454",
|
|
"openssl-cve-2022-3786",
|
|
"openssl-cve-2022-3786-ossl_punycode_decode",
|
|
"openssl-cve-2022-3996",
|
|
"openssl-cve-2022-4304",
|
|
"openssl-cve-2022-4304-bn_blinding_invert_ex",
|
|
"openssl-cve-2022-4304-rsa_blinding_invert",
|
|
"openssl-cve-2022-4304-rsa_ossl_private_decrypt",
|
|
"openssl-cve-2023-0216",
|
|
"openssl-cve-2023-0217",
|
|
"openssl-cve-2023-0286",
|
|
"openssl-cve-2023-0286-general_name_cmp",
|
|
"openssl-cve-2023-0401",
|
|
"openssl-cve-2023-0464",
|
|
"openssl-cve-2023-0464-tree_init",
|
|
"openssl-cve-2023-0465",
|
|
"openssl-cve-2023-2650",
|
|
"openssl-cve-2023-2975",
|
|
"openssl-cve-2023-3446",
|
|
"openssl-cve-2023-3817-dh_check",
|
|
"openssl-cve-2023-5363",
|
|
"openssl-cve-2024-0727",
|
|
"openssl-cve-2024-0727-smime_write_pkcs7",
|
|
"openssl-cve-2024-12797",
|
|
"openssl-cve-2024-13176-ec_field_inverse_mod_ord",
|
|
"openssl-cve-2024-2511",
|
|
"openssl-cve-2024-2511-tls_construct_server_hel",
|
|
"openssl-cve-2024-4603",
|
|
"openssl-cve-2024-4741",
|
|
"openssl-cve-2024-4741-ssl3_release_read_buffer",
|
|
"openssl-cve-2024-4741-ssl_free_buffers",
|
|
"openssl-cve-2024-4741-tls_free_buffers",
|
|
"openssl-cve-2024-4741-tls_release_read_buffer",
|
|
"openssl-cve-2024-5535",
|
|
"openssl-cve-2024-5535-alpn_select_cb",
|
|
"openssl-cve-2024-5535-ssl_select_next_proto",
|
|
"openssl-cve-2024-5535-tls_construct_stoc_next_",
|
|
"openssl-cve-2024-5535-tls_parse_stoc_npn",
|
|
"openssl-cve-2024-6119",
|
|
"openssl-cve-2025-11187",
|
|
"openssl-cve-2025-15467",
|
|
"openssl-cve-2025-15469",
|
|
"openssl-cve-2025-66199",
|
|
"openssl-cve-2025-66199-tls13_process_compressed",
|
|
"openssl-cve-2025-68160",
|
|
"openssl-cve-2025-69418",
|
|
"openssl-cve-2025-69419",
|
|
"openssl-cve-2025-69421",
|
|
"openssl-cve-2025-9230",
|
|
"openssl-cve-2025-9232",
|
|
"openssl-cve-2026-14456-expect_quic_for_value",
|
|
"openssl-cve-2026-14456-port_default_packet_hand",
|
|
"openssl-cve-2026-14457",
|
|
"openssl-cve-2026-18798",
|
|
"openssl-cve-2026-2673",
|
|
"openssl-cve-2026-28388",
|
|
"openssl-cve-2026-28389",
|
|
"openssl-cve-2026-28389-dh_cms_set_shared_info",
|
|
"openssl-cve-2026-28389-ecdh_cms_set_shared_info",
|
|
"openssl-cve-2026-31789",
|
|
"openssl-cve-2026-31790",
|
|
"openssl-cve-2026-31790-rsasve_generate",
|
|
"openssl-cve-2026-34181",
|
|
"openssl-cve-2026-35188",
|
|
"openssl-cve-2026-42764",
|
|
"openssl-cve-2026-42765",
|
|
"openssl-cve-2026-42766",
|
|
"openssl-cve-2026-42768",
|
|
"openssl-cve-2026-42768-pkcs7_decrypt_rinfo",
|
|
"openssl-cve-2026-42769",
|
|
"openssl-cve-2026-42771",
|
|
"openssl-cve-2026-45445",
|
|
"openssl-cve-2026-45446",
|
|
"openssl-cve-2026-45446-aes_siv_set_ctx_params",
|
|
"openssl-cve-2026-45447",
|
|
"openssl-cve-2026-54874",
|
|
"openssl-cve-2026-54874-dtls1_process_record",
|
|
"openssl-cve-2026-54874-dtls_rlayer_buffer_recor",
|
|
"openssl-cve-2026-63072",
|
|
"openssl-cve-2026-63072-cms_kek_cipher",
|
|
"openssl-cve-2026-63073",
|
|
"openssl-cve-2026-63074",
|
|
"openssl-cve-2026-63075-txp_pkt_commit",
|
|
"openssl-cve-2026-7383",
|
|
"openssl-cve-2026-75803",
|
|
"openssl-cve-2026-75803-aes_gcm_siv_finish",
|
|
"openssl-cve-2026-75803-chacha20_poly1305_cipher",
|
|
"openssl-cve-2026-9076"
|
|
]
|
|
],
|
|
[
|
|
"sgabe/DSVPWA",
|
|
[
|
|
"dsvpwa-attacks-run",
|
|
"dsvpwa-dsvpwa-main",
|
|
"dsvpwa-handlers-do-bdr"
|
|
]
|
|
]
|
|
],
|
|
"undeclared_split": [
|
|
[
|
|
"local-python-vulnerable",
|
|
"holdout"
|
|
],
|
|
[
|
|
"local-javascript-vulnerable",
|
|
"holdout"
|
|
],
|
|
[
|
|
"local-cpp-buffer",
|
|
"train"
|
|
]
|
|
],
|
|
"vendored_pairs": 317
|
|
}
|