mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(learning-harness): corpus hygiene at load, with the defects it finds
Task 1.3. An expected row may carry a family label, stored opaquely because the classifier owns the taxonomy and the corpus module never imports the learning package. Every pair carries a computed reason when it cannot be scored: a declared known limit wins, otherwise identical twin when the two excerpts are the same code once the provenance header is stripped. No row is ever dropped. Two corpus defects the new accounting surfaces, committed as an artifact beside the numbers they cap. Nineteen repositories sit on both sides of the split, so a learning path could be taught by one file of a repository and evaluated on another. Ten groups of pairs share a vulnerable excerpt byte for byte, mostly the same fix harvested twice at different granularity. The split helper keeps every declared split, lets rows that named none inherit their repository's side, and otherwise assigns a whole repository at once, oldest first, so nothing straddles by our doing. Pair signals can now be restricted to one split, which is what lets a learning path teach from train rows only. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5
Claude Opus 5
parent
944df86d12
commit
6c205b728f
@@ -0,0 +1,389 @@
|
|||||||
|
{
|
||||||
|
"command": "python -c 'from openultrasast.pairs import ...; duplicate_groups / split_by_repository over the vendored catalog'",
|
||||||
|
"duplicate_groups": [
|
||||||
|
[
|
||||||
|
"openssl-cve-2023-0286",
|
||||||
|
"openssl-cve-2023-0286-general_name_cmp"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"openniw-jobs-0fc947",
|
||||||
|
"openniw-jobs-b74de3"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"curl-cve-2023-38545",
|
||||||
|
"curl-cve-2023-38545-do_socks5"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"openssl-cve-2021-3712",
|
||||||
|
"openssl-cve-2021-3712-append_ia5"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"openssl-cve-2020-1967",
|
||||||
|
"openssl-cve-2020-1967-tls1_check_sig_alg"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"openssl-cve-2022-3786",
|
||||||
|
"openssl-cve-2022-3786-ossl_punycode_decode"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"openssl-cve-2020-1971",
|
||||||
|
"openssl-cve-2020-1971-general_name_cmp"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"etude-lexical-markdown-editorconfig-c61a8b",
|
||||||
|
"popmark-editorconfig-e5ac40"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"anastasios-pw-session-14ec1a",
|
||||||
|
"astroclaw-pw-session-1553a8"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"openssl-cve-2022-3602",
|
||||||
|
"openssl-cve-2022-3602-ossl_punycode_decode"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"generated": "2026-09-06",
|
||||||
|
"identical_twins": [
|
||||||
|
"dsvw-dsvw-do-get",
|
||||||
|
"python-insecure-app-main-try-hack-me",
|
||||||
|
"pythonssti-main-read-root",
|
||||||
|
"vampi-books-get-by-title",
|
||||||
|
"vulnpy-deserialization-do-pickle-load"
|
||||||
|
],
|
||||||
|
"known_limit": [
|
||||||
|
"owasp-java-hash"
|
||||||
|
],
|
||||||
|
"note": "Repositories in `straddling` have pairs on both sides of the train/holdout split, so a learning path can be taught by one file of a repository and evaluated on another. Pairs in `duplicate_groups` share a vulnerable excerpt byte for byte after the provenance header is stripped. Both cap what any detector number can mean.",
|
||||||
|
"straddling": [
|
||||||
|
[
|
||||||
|
"Contrast-Security-OSS/DjanGoat",
|
||||||
|
[
|
||||||
|
"djangoat-analytics-hits-by-ip",
|
||||||
|
"djangoat-views-reset-password",
|
||||||
|
"djangoat-views-user-view"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"Contrast-Security-OSS/vulnpy",
|
||||||
|
[
|
||||||
|
"vulnpy-cmdi-do-os-system",
|
||||||
|
"vulnpy-cmdi-do-subprocess-popen",
|
||||||
|
"vulnpy-deserialization-do-pickle-load"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"LinuxUser255/Vulnerable_Python_Apps",
|
||||||
|
[
|
||||||
|
"vulnerable-python-apps-bad-mvc-do-get",
|
||||||
|
"vulnerable-python-apps-dir-traversal-do-get"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"OWASP-Benchmark/BenchmarkJava",
|
||||||
|
[
|
||||||
|
"owasp-java-cmdi",
|
||||||
|
"owasp-java-hash",
|
||||||
|
"owasp-java-sqli"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"OWASP-Benchmark/BenchmarkPython",
|
||||||
|
[
|
||||||
|
"owasp-python-cmdi",
|
||||||
|
"owasp-python-codeinj",
|
||||||
|
"owasp-python-deser",
|
||||||
|
"owasp-python-hash",
|
||||||
|
"owasp-python-sqli"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"anotherik/ThreatByte",
|
||||||
|
[
|
||||||
|
"threatbyte-api-v1-delete",
|
||||||
|
"threatbyte-api-v1-get",
|
||||||
|
"threatbyte-routes-dashboard"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"anxolerd/dvpwa",
|
||||||
|
[
|
||||||
|
"dvpwa-app-init",
|
||||||
|
"dvpwa-middlewares-session-middleware",
|
||||||
|
"dvpwa-student-create"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"arichardson/juliet-test-suite-c",
|
||||||
|
[
|
||||||
|
"juliet-c-cwe121-strcpy",
|
||||||
|
"juliet-c-cwe134-printf"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"bigfivefoods/supplieradvisor-mvp",
|
||||||
|
[
|
||||||
|
"supplieradvisor-mvp-route-387bcf",
|
||||||
|
"supplieradvisor-mvp-route-a69e8f"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"chromium/chromium",
|
||||||
|
[
|
||||||
|
"chromium-cve-2019-5786",
|
||||||
|
"chromium-cve-2022-2162"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"curl/curl",
|
||||||
|
[
|
||||||
|
"curl-cve-2020-8177",
|
||||||
|
"curl-cve-2020-8231-curl_multi_add_handle",
|
||||||
|
"curl-cve-2020-8231-curl_open",
|
||||||
|
"curl-cve-2020-8284",
|
||||||
|
"curl-cve-2020-8284-config_init",
|
||||||
|
"curl-cve-2020-8286",
|
||||||
|
"curl-cve-2021-22876",
|
||||||
|
"curl-cve-2021-22890",
|
||||||
|
"curl-cve-2021-22898",
|
||||||
|
"curl-cve-2021-22901-ossl_connect_step1",
|
||||||
|
"curl-cve-2021-22924",
|
||||||
|
"curl-cve-2021-22924-nss_do_connect",
|
||||||
|
"curl-cve-2021-22924-servercert",
|
||||||
|
"curl-cve-2021-22925",
|
||||||
|
"curl-cve-2021-22945",
|
||||||
|
"curl-cve-2021-22946",
|
||||||
|
"curl-cve-2021-22946-imap_state_capability_re",
|
||||||
|
"curl-cve-2021-22946-pop3_state_capa_resp",
|
||||||
|
"curl-cve-2021-22947",
|
||||||
|
"curl-cve-2021-22947-imap_state_starttls_resp",
|
||||||
|
"curl-cve-2021-22947-pop3_state_starttls_resp",
|
||||||
|
"curl-cve-2021-22947-smtp_state_starttls_resp",
|
||||||
|
"curl-cve-2022-22576-conn_free",
|
||||||
|
"curl-cve-2022-27774",
|
||||||
|
"curl-cve-2022-27775",
|
||||||
|
"curl-cve-2022-27776",
|
||||||
|
"curl-cve-2022-27778",
|
||||||
|
"curl-cve-2022-27780",
|
||||||
|
"curl-cve-2022-27782-curl_free_primary_ssl_co",
|
||||||
|
"curl-cve-2022-27782-curl_init_userdefined",
|
||||||
|
"curl-cve-2022-27782-gtls_shutdown",
|
||||||
|
"curl-cve-2022-27782-nss_setup_connect",
|
||||||
|
"curl-cve-2022-27782-ossl_connect_step1",
|
||||||
|
"curl-cve-2022-30115",
|
||||||
|
"curl-cve-2022-32206",
|
||||||
|
"curl-cve-2022-32207",
|
||||||
|
"curl-cve-2022-32208",
|
||||||
|
"curl-cve-2023-38545",
|
||||||
|
"curl-cve-2023-38545-do_socks5",
|
||||||
|
"curl-cve-2026-9080"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"dayflower/etude-lexical-markdown",
|
||||||
|
[
|
||||||
|
"etude-lexical-markdown-editorconfig-c61a8b",
|
||||||
|
"etude-lexical-markdown-editorconfig-db4f54"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"dolevf/Damn-Vulnerable-GraphQL-Application",
|
||||||
|
[
|
||||||
|
"damn-vulnerable-graphql-application-views-mutate",
|
||||||
|
"damn-vulnerable-graphql-application-views-resolve-pastes",
|
||||||
|
"damn-vulnerable-graphql-application-views-resolve-system-dia"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"erev0s/VAmPI",
|
||||||
|
[
|
||||||
|
"vampi-books-get-by-title",
|
||||||
|
"vampi-user-model-get-user",
|
||||||
|
"vampi-users-update-password"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"fportantier/vulpy",
|
||||||
|
[
|
||||||
|
"vulpy-libuser-create",
|
||||||
|
"vulpy-libuser-login",
|
||||||
|
"vulpy-libuser-password-change"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"manuelz120/extremely-vulnerable-flask-app",
|
||||||
|
[
|
||||||
|
"extremely-vulnerable-flask-app-account-before-request",
|
||||||
|
"extremely-vulnerable-flask-app-account-search",
|
||||||
|
"extremely-vulnerable-flask-app-signup-validate-token"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"naryal2580/vfapi",
|
||||||
|
[
|
||||||
|
"vfapi-main-delete-user",
|
||||||
|
"vfapi-main-put-user",
|
||||||
|
"vfapi-main-sql-return-users-from-username"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"openssl/openssl",
|
||||||
|
[
|
||||||
|
"openssl-cve-2014-0160",
|
||||||
|
"openssl-cve-2020-1967",
|
||||||
|
"openssl-cve-2020-1967-tls1_check_sig_alg",
|
||||||
|
"openssl-cve-2020-1971",
|
||||||
|
"openssl-cve-2020-1971-general_name_cmp",
|
||||||
|
"openssl-cve-2020-25659",
|
||||||
|
"openssl-cve-2021-23840",
|
||||||
|
"openssl-cve-2021-23840-evp_encryptdecryptupdate",
|
||||||
|
"openssl-cve-2021-23841",
|
||||||
|
"openssl-cve-2021-3449",
|
||||||
|
"openssl-cve-2021-3449-def_generate_session_id",
|
||||||
|
"openssl-cve-2021-3449-init_sig_algs",
|
||||||
|
"openssl-cve-2021-3449-init_sig_algs_cert",
|
||||||
|
"openssl-cve-2021-3449-ssl_ctx_set_alpn_protos",
|
||||||
|
"openssl-cve-2021-3449-ssl_generate_master_secr",
|
||||||
|
"openssl-cve-2021-3449-tls_construct_ctos_early",
|
||||||
|
"openssl-cve-2021-3449-tls_handle_alpn",
|
||||||
|
"openssl-cve-2021-3449-tls_parse_ctos_renegotia",
|
||||||
|
"openssl-cve-2021-3449-tls_process_certificate_",
|
||||||
|
"openssl-cve-2021-3449-tls_process_finished",
|
||||||
|
"openssl-cve-2021-3450",
|
||||||
|
"openssl-cve-2021-3711",
|
||||||
|
"openssl-cve-2021-3711-pkey_sm2_decrypt",
|
||||||
|
"openssl-cve-2021-3711-sm2_asym_decrypt",
|
||||||
|
"openssl-cve-2021-3712",
|
||||||
|
"openssl-cve-2021-3712-append_ia5",
|
||||||
|
"openssl-cve-2021-3712-general_name_print",
|
||||||
|
"openssl-cve-2021-3712-i2r_pci",
|
||||||
|
"openssl-cve-2021-3712-ia5ncasecmp",
|
||||||
|
"openssl-cve-2021-3712-netscape_spki_print",
|
||||||
|
"openssl-cve-2021-3712-ossl_cmp_hdr_push1_freet",
|
||||||
|
"openssl-cve-2021-3712-print_qualifiers",
|
||||||
|
"openssl-cve-2021-4044",
|
||||||
|
"openssl-cve-2021-4044-tls_post_process_server_",
|
||||||
|
"openssl-cve-2022-0778",
|
||||||
|
"openssl-cve-2022-1292",
|
||||||
|
"openssl-cve-2022-1343",
|
||||||
|
"openssl-cve-2022-1434",
|
||||||
|
"openssl-cve-2022-2068-hash_dir",
|
||||||
|
"openssl-cve-2022-29242",
|
||||||
|
"openssl-cve-2022-3358",
|
||||||
|
"openssl-cve-2022-3358-evp_cipher_init_internal",
|
||||||
|
"openssl-cve-2022-3602",
|
||||||
|
"openssl-cve-2022-3602-ossl_punycode_decode",
|
||||||
|
"openssl-cve-2022-37454",
|
||||||
|
"openssl-cve-2022-3786",
|
||||||
|
"openssl-cve-2022-3786-ossl_punycode_decode",
|
||||||
|
"openssl-cve-2022-3996",
|
||||||
|
"openssl-cve-2022-4304",
|
||||||
|
"openssl-cve-2022-4304-bn_blinding_invert_ex",
|
||||||
|
"openssl-cve-2022-4304-rsa_blinding_invert",
|
||||||
|
"openssl-cve-2022-4304-rsa_ossl_private_decrypt",
|
||||||
|
"openssl-cve-2023-0216",
|
||||||
|
"openssl-cve-2023-0217",
|
||||||
|
"openssl-cve-2023-0286",
|
||||||
|
"openssl-cve-2023-0286-general_name_cmp",
|
||||||
|
"openssl-cve-2023-0401",
|
||||||
|
"openssl-cve-2023-0464",
|
||||||
|
"openssl-cve-2023-0464-tree_init",
|
||||||
|
"openssl-cve-2023-0465",
|
||||||
|
"openssl-cve-2023-2650",
|
||||||
|
"openssl-cve-2023-2975",
|
||||||
|
"openssl-cve-2023-3446",
|
||||||
|
"openssl-cve-2023-3817-dh_check",
|
||||||
|
"openssl-cve-2023-5363",
|
||||||
|
"openssl-cve-2024-0727",
|
||||||
|
"openssl-cve-2024-0727-smime_write_pkcs7",
|
||||||
|
"openssl-cve-2024-12797",
|
||||||
|
"openssl-cve-2024-13176-ec_field_inverse_mod_ord",
|
||||||
|
"openssl-cve-2024-2511",
|
||||||
|
"openssl-cve-2024-2511-tls_construct_server_hel",
|
||||||
|
"openssl-cve-2024-4603",
|
||||||
|
"openssl-cve-2024-4741",
|
||||||
|
"openssl-cve-2024-4741-ssl3_release_read_buffer",
|
||||||
|
"openssl-cve-2024-4741-ssl_free_buffers",
|
||||||
|
"openssl-cve-2024-4741-tls_free_buffers",
|
||||||
|
"openssl-cve-2024-4741-tls_release_read_buffer",
|
||||||
|
"openssl-cve-2024-5535",
|
||||||
|
"openssl-cve-2024-5535-alpn_select_cb",
|
||||||
|
"openssl-cve-2024-5535-ssl_select_next_proto",
|
||||||
|
"openssl-cve-2024-5535-tls_construct_stoc_next_",
|
||||||
|
"openssl-cve-2024-5535-tls_parse_stoc_npn",
|
||||||
|
"openssl-cve-2024-6119",
|
||||||
|
"openssl-cve-2025-11187",
|
||||||
|
"openssl-cve-2025-15467",
|
||||||
|
"openssl-cve-2025-15469",
|
||||||
|
"openssl-cve-2025-66199",
|
||||||
|
"openssl-cve-2025-66199-tls13_process_compressed",
|
||||||
|
"openssl-cve-2025-68160",
|
||||||
|
"openssl-cve-2025-69418",
|
||||||
|
"openssl-cve-2025-69419",
|
||||||
|
"openssl-cve-2025-69421",
|
||||||
|
"openssl-cve-2025-9230",
|
||||||
|
"openssl-cve-2025-9232",
|
||||||
|
"openssl-cve-2026-14456-expect_quic_for_value",
|
||||||
|
"openssl-cve-2026-14456-port_default_packet_hand",
|
||||||
|
"openssl-cve-2026-14457",
|
||||||
|
"openssl-cve-2026-18798",
|
||||||
|
"openssl-cve-2026-2673",
|
||||||
|
"openssl-cve-2026-28388",
|
||||||
|
"openssl-cve-2026-28389",
|
||||||
|
"openssl-cve-2026-28389-dh_cms_set_shared_info",
|
||||||
|
"openssl-cve-2026-28389-ecdh_cms_set_shared_info",
|
||||||
|
"openssl-cve-2026-31789",
|
||||||
|
"openssl-cve-2026-31790",
|
||||||
|
"openssl-cve-2026-31790-rsasve_generate",
|
||||||
|
"openssl-cve-2026-34181",
|
||||||
|
"openssl-cve-2026-35188",
|
||||||
|
"openssl-cve-2026-42764",
|
||||||
|
"openssl-cve-2026-42765",
|
||||||
|
"openssl-cve-2026-42766",
|
||||||
|
"openssl-cve-2026-42768",
|
||||||
|
"openssl-cve-2026-42768-pkcs7_decrypt_rinfo",
|
||||||
|
"openssl-cve-2026-42769",
|
||||||
|
"openssl-cve-2026-42771",
|
||||||
|
"openssl-cve-2026-45445",
|
||||||
|
"openssl-cve-2026-45446",
|
||||||
|
"openssl-cve-2026-45446-aes_siv_set_ctx_params",
|
||||||
|
"openssl-cve-2026-45447",
|
||||||
|
"openssl-cve-2026-54874",
|
||||||
|
"openssl-cve-2026-54874-dtls1_process_record",
|
||||||
|
"openssl-cve-2026-54874-dtls_rlayer_buffer_recor",
|
||||||
|
"openssl-cve-2026-63072",
|
||||||
|
"openssl-cve-2026-63072-cms_kek_cipher",
|
||||||
|
"openssl-cve-2026-63073",
|
||||||
|
"openssl-cve-2026-63074",
|
||||||
|
"openssl-cve-2026-63075-txp_pkt_commit",
|
||||||
|
"openssl-cve-2026-7383",
|
||||||
|
"openssl-cve-2026-75803",
|
||||||
|
"openssl-cve-2026-75803-aes_gcm_siv_finish",
|
||||||
|
"openssl-cve-2026-75803-chacha20_poly1305_cipher",
|
||||||
|
"openssl-cve-2026-9076"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"sgabe/DSVPWA",
|
||||||
|
[
|
||||||
|
"dsvpwa-attacks-run",
|
||||||
|
"dsvpwa-dsvpwa-main",
|
||||||
|
"dsvpwa-handlers-do-bdr"
|
||||||
|
]
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"undeclared_split": [
|
||||||
|
[
|
||||||
|
"local-python-vulnerable",
|
||||||
|
"holdout"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"local-javascript-vulnerable",
|
||||||
|
"holdout"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"local-cpp-buffer",
|
||||||
|
"train"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"vendored_pairs": 277
|
||||||
|
}
|
||||||
@@ -28,6 +28,7 @@ class ExpectedFinding:
|
|||||||
sink: str | None = None
|
sink: str | None = None
|
||||||
mechanism: str | None = None
|
mechanism: str | None = None
|
||||||
obligation: str | None = None # additive (authorization-obligations, Req 7.2): an OPERATION_KIND the labeled function owes
|
obligation: str | None = None # additive (authorization-obligations, Req 7.2): an OPERATION_KIND the labeled function owes
|
||||||
|
family: str | None = None # additive (learning-harness, Req 4): an opaque family id; the classifier owns the taxonomy
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ evaluated but excluded from the achievable numbers.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import math
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -126,6 +128,9 @@ class PairCase:
|
|||||||
review_tier: str = "advisory"
|
review_tier: str = "advisory"
|
||||||
reviewer: str = ""
|
reviewer: str = ""
|
||||||
vendored: bool = True
|
vendored: bool = True
|
||||||
|
unscorable: str | None = None # computed: identical_twin, known_limit:<reason> (learning-harness Req 4.1)
|
||||||
|
split_declared: bool = True # False when the catalog row named no split, so the split helper may assign one
|
||||||
|
fix_date: str = "" # ISO date of the fixing commit; the catalog generator fills it, the split helper orders by it
|
||||||
recipe: tuple[tuple[str, object], ...] = () # pointer pairs carry their harvest recipe instead of excerpts
|
recipe: tuple[tuple[str, object], ...] = () # pointer pairs carry their harvest recipe instead of excerpts
|
||||||
|
|
||||||
|
|
||||||
@@ -276,6 +281,9 @@ def _load_catalog_file(path: Path) -> tuple[PairCase, ...]:
|
|||||||
provenance=provenance,
|
provenance=provenance,
|
||||||
split=split,
|
split=split,
|
||||||
known_limit=str(known_limit) if known_limit else None,
|
known_limit=str(known_limit) if known_limit else None,
|
||||||
|
unscorable=_unscorable_reason(known_limit, vuln_file, fixed_file, vendored=vendored),
|
||||||
|
split_declared="split" in item,
|
||||||
|
fix_date=str(item.get("fix_date", "")),
|
||||||
review_tier=review_tier,
|
review_tier=review_tier,
|
||||||
reviewer=reviewer,
|
reviewer=reviewer,
|
||||||
vendored=vendored,
|
vendored=vendored,
|
||||||
@@ -459,12 +467,17 @@ def evaluate_catalog(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def build_pair_signals(outcomes: Sequence[PairOutcome]) -> list[dict[str, object]]:
|
def build_pair_signals(outcomes: Sequence[PairOutcome], *, split: str | None = None) -> list[dict[str, object]]:
|
||||||
"""Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name, profile, and mechanisms."""
|
"""Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name, profile, and mechanisms.
|
||||||
|
|
||||||
|
``split`` restricts the signals to one split, so a learning path teaches from the train rows only (Req 5.1).
|
||||||
|
"""
|
||||||
signals: list[dict[str, object]] = []
|
signals: list[dict[str, object]] = []
|
||||||
for outcome in outcomes:
|
for outcome in outcomes:
|
||||||
if outcome.known_limit is not None:
|
if outcome.known_limit is not None:
|
||||||
continue # unachievable by construction; the loop must not chase it
|
continue # unachievable by construction; the loop must not chase it
|
||||||
|
if split is not None and outcome.split != split:
|
||||||
|
continue
|
||||||
common: dict[str, object] = {
|
common: dict[str, object] = {
|
||||||
"pair": outcome.name,
|
"pair": outcome.name,
|
||||||
"profile": outcome.provenance,
|
"profile": outcome.provenance,
|
||||||
@@ -480,6 +493,87 @@ def build_pair_signals(outcomes: Sequence[PairOutcome]) -> list[dict[str, object
|
|||||||
return sorted(signals, key=lambda item: (str(item.get("signal")), str(item.get("rule_id")), str(item.get("pair"))))
|
return sorted(signals, key=lambda item: (str(item.get("signal")), str(item.get("rule_id")), str(item.get("pair"))))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SplitReport:
|
||||||
|
"""What ``split_by_repository`` had to do and what it found (learning-harness Req 4.4)."""
|
||||||
|
|
||||||
|
assigned: tuple[tuple[str, str], ...] = () # (pair, split) for rows whose catalog row named none
|
||||||
|
straddling: tuple[tuple[str, tuple[str, ...]], ...] = () # repository -> pairs, when one repository sits in both splits
|
||||||
|
undated: int = 0 # assigned rows with no fix date, ordered by name instead
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=4096)
|
||||||
|
def _body_digest(path: Path) -> str | None:
|
||||||
|
"""Hash of an excerpt with its provenance header and trailing whitespace removed; None when unreadable or empty."""
|
||||||
|
try:
|
||||||
|
text = path.read_text(errors="ignore")
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
lines = text.splitlines()
|
||||||
|
index = 0
|
||||||
|
while index < len(lines) and (not lines[index].strip() or lines[index].lstrip().startswith(("#", "//", "/*", "*"))):
|
||||||
|
index += 1
|
||||||
|
body = "\n".join(line.rstrip() for line in lines[index:]).strip()
|
||||||
|
return hashlib.sha256(body.encode()).hexdigest() if body else None
|
||||||
|
|
||||||
|
|
||||||
|
def _unscorable_reason(known_limit: object, vuln_file: Path, fixed_file: Path, *, vendored: bool) -> str | None:
|
||||||
|
"""Why this pair cannot be scored, or None. A declared known limit wins over anything computed."""
|
||||||
|
if known_limit:
|
||||||
|
return f"known_limit:{known_limit}"
|
||||||
|
if not vendored:
|
||||||
|
return None
|
||||||
|
digest = _body_digest(vuln_file)
|
||||||
|
return "identical_twin" if digest is not None and digest == _body_digest(fixed_file) else None
|
||||||
|
|
||||||
|
|
||||||
|
def duplicate_groups(cases: Sequence[PairCase]) -> tuple[tuple[str, ...], ...]:
|
||||||
|
"""Pairs whose vulnerable excerpts are the same code, reported before any split is applied (Req 4.3)."""
|
||||||
|
by_digest: dict[str, list[str]] = {}
|
||||||
|
for case in cases:
|
||||||
|
digest = _body_digest(case.vuln_file) if case.vendored else None
|
||||||
|
if digest is not None:
|
||||||
|
by_digest.setdefault(digest, []).append(case.name)
|
||||||
|
return tuple(tuple(sorted(names)) for _, names in sorted(by_digest.items()) if len(names) > 1)
|
||||||
|
|
||||||
|
|
||||||
|
def split_by_repository(cases: Sequence[PairCase], *, holdout_fraction: float = 0.5) -> tuple[tuple[PairCase, ...], SplitReport]:
|
||||||
|
"""Keep every declared split, assign the rest by repository in date order, and report repositories that straddle.
|
||||||
|
|
||||||
|
A repository never straddles by our doing: rows that named no split inherit their repository's declared split when
|
||||||
|
there is one, and otherwise the whole repository lands on one side, oldest first (Req 4.4).
|
||||||
|
"""
|
||||||
|
groups: dict[str, list[PairCase]] = {}
|
||||||
|
for case in cases:
|
||||||
|
groups.setdefault(case.repo or f"{case.slice}:{case.name}", []).append(case)
|
||||||
|
straddling = tuple(
|
||||||
|
(repo, tuple(sorted(case.name for case in members)))
|
||||||
|
for repo, members in sorted(groups.items())
|
||||||
|
if len({case.split for case in members if case.split_declared}) > 1
|
||||||
|
)
|
||||||
|
open_repos = [repo for repo, members in groups.items() if any(not case.split_declared for case in members)]
|
||||||
|
inherited = {repo: next((case.split for case in groups[repo] if case.split_declared), None) for repo in open_repos}
|
||||||
|
orderable = sorted(
|
||||||
|
(repo for repo in open_repos if inherited[repo] is None),
|
||||||
|
key=lambda repo: (min((case.fix_date for case in groups[repo] if case.fix_date), default="9999-99-99"), repo),
|
||||||
|
)
|
||||||
|
holdout_count = math.ceil(len(orderable) * holdout_fraction) if orderable else 0
|
||||||
|
holdout = set(orderable[len(orderable) - holdout_count :]) if holdout_count else set()
|
||||||
|
assigned: list[tuple[str, str]] = []
|
||||||
|
undated = 0
|
||||||
|
out: list[PairCase] = []
|
||||||
|
for case in cases:
|
||||||
|
repo = case.repo or f"{case.slice}:{case.name}"
|
||||||
|
if case.split_declared:
|
||||||
|
out.append(case)
|
||||||
|
continue
|
||||||
|
split = inherited.get(repo) or ("holdout" if repo in holdout else "train")
|
||||||
|
undated += 0 if case.fix_date else 1
|
||||||
|
assigned.append((case.name, split))
|
||||||
|
out.append(replace(case, split=split, split_declared=True))
|
||||||
|
return tuple(out), SplitReport(assigned=tuple(assigned), straddling=straddling, undated=undated)
|
||||||
|
|
||||||
|
|
||||||
def select_slice(cases: Sequence[PairCase], slice_name: str | None) -> tuple[PairCase, ...]:
|
def select_slice(cases: Sequence[PairCase], slice_name: str | None) -> tuple[PairCase, ...]:
|
||||||
if not slice_name or slice_name == "all":
|
if not slice_name or slice_name == "all":
|
||||||
return tuple(cases)
|
return tuple(cases)
|
||||||
@@ -558,6 +652,7 @@ def _parse_expected(item: dict[str, object], name: str, mechanisms: frozenset[st
|
|||||||
raise CatalogError(f"pair {name}: expected row is missing mechanism")
|
raise CatalogError(f"pair {name}: expected row is missing mechanism")
|
||||||
if mechanism not in mechanisms:
|
if mechanism not in mechanisms:
|
||||||
raise CatalogError(f"pair {name}: unknown mechanism {mechanism!r}; see benchmarks/pairs/mechanisms.toml")
|
raise CatalogError(f"pair {name}: unknown mechanism {mechanism!r}; see benchmarks/pairs/mechanisms.toml")
|
||||||
|
family = item.get("family")
|
||||||
obligation = item.get("obligation")
|
obligation = item.get("obligation")
|
||||||
if obligation is not None and obligation not in OPERATION_KINDS:
|
if obligation is not None and obligation not in OPERATION_KINDS:
|
||||||
raise CatalogError(f"pair {name}: unknown obligation {obligation!r}; expected one of {sorted(OPERATION_KINDS)}")
|
raise CatalogError(f"pair {name}: unknown obligation {obligation!r}; expected one of {sorted(OPERATION_KINDS)}")
|
||||||
@@ -572,6 +667,7 @@ def _parse_expected(item: dict[str, object], name: str, mechanisms: frozenset[st
|
|||||||
sink=str(item["sink"]) if "sink" in item else None,
|
sink=str(item["sink"]) if "sink" in item else None,
|
||||||
mechanism=mechanism,
|
mechanism=mechanism,
|
||||||
obligation=str(obligation) if obligation is not None else None,
|
obligation=str(obligation) if obligation is not None else None,
|
||||||
|
family=str(family) if family is not None else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
"""learning-harness task 1.3: additive corpus fields and hygiene at load (offline)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored
|
||||||
|
|
||||||
|
IDENTICAL_TWINS = {
|
||||||
|
"dsvw-dsvw-do-get",
|
||||||
|
"python-insecure-app-main-try-hack-me",
|
||||||
|
"pythonssti-main-read-root",
|
||||||
|
"vampi-books-get-by-title",
|
||||||
|
"vulnpy-deserialization-do-pickle-load",
|
||||||
|
}
|
||||||
|
VENDORED_PER_SLICE = {"local": 3, "github": 6, "sast": 11, "vibe-py": 35, "vfc-js": 17, "vfc": 176, "agent-vfc": 29}
|
||||||
|
|
||||||
|
|
||||||
|
def test_identical_twins_are_unscorable_and_no_row_is_dropped() -> None:
|
||||||
|
cases = load_pair_catalog(DEFAULT_CATALOG)
|
||||||
|
per_slice: dict[str, int] = {}
|
||||||
|
for case in select_vendored(cases):
|
||||||
|
per_slice[case.slice] = per_slice.get(case.slice, 0) + 1
|
||||||
|
assert per_slice == VENDORED_PER_SLICE # nothing dropped by the new accounting
|
||||||
|
twins = {case.name for case in cases if case.unscorable == "identical_twin"}
|
||||||
|
assert twins == IDENTICAL_TWINS
|
||||||
|
assert all(case.unscorable is None for case in cases if case.name not in IDENTICAL_TWINS and not case.known_limit)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_declared_known_limit_wins_over_a_computed_reason(tmp_path: Path) -> None:
|
||||||
|
from openultrasast.pairs import _load_catalog_file
|
||||||
|
|
||||||
|
(tmp_path / "v.py").write_text("# provenance: x\n\ndef f():\n return 1\n")
|
||||||
|
(tmp_path / "f.py").write_text("# provenance: y\n\ndef f():\n return 1\n")
|
||||||
|
head = (
|
||||||
|
'[[pair]]\nname = "t"\nslice = "sast"\nvuln = "v.py"\nfixed = "f.py"\nrelpath = "app.py"\n{extra}\n'
|
||||||
|
'[[pair.expected]]\ncwe = "CWE-95"\nclass = "x"\npath = "app.py"\nsink = "eval"\nmechanism = "other"\n'
|
||||||
|
)
|
||||||
|
(tmp_path / "plain.toml").write_text(head.format(extra=""))
|
||||||
|
(case,) = _load_catalog_file(tmp_path / "plain.toml")
|
||||||
|
assert case.unscorable == "identical_twin" # the provenance headers differ, the bodies do not
|
||||||
|
(tmp_path / "limited.toml").write_text(head.format(extra='known_limit = "java hash"'))
|
||||||
|
(limited,) = _load_catalog_file(tmp_path / "limited.toml")
|
||||||
|
assert limited.unscorable == "known_limit:java hash"
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_rows_carry_an_opaque_family_label(tmp_path: Path) -> None:
|
||||||
|
from openultrasast.pairs import _load_catalog_file
|
||||||
|
|
||||||
|
(tmp_path / "v.py").write_text("x = 1\n")
|
||||||
|
(tmp_path / "f.py").write_text("x = 2\n")
|
||||||
|
row = (
|
||||||
|
'[[pair]]\nname = "t"\nslice = "sast"\nvuln = "v.py"\nfixed = "f.py"\nrelpath = "app.py"\n\n'
|
||||||
|
'[[pair.expected]]\ncwe = "CWE-95"\nclass = "x"\npath = "app.py"\nsink = "eval"\nmechanism = "other"\nfamily = "{family}"\n'
|
||||||
|
)
|
||||||
|
(tmp_path / "c.toml").write_text(row.format(family="injection"))
|
||||||
|
(case,) = _load_catalog_file(tmp_path / "c.toml")
|
||||||
|
assert case.expected[0].family == "injection"
|
||||||
|
# the corpus stores the label; the classifier owns the taxonomy, so a fabricated id is not rejected here
|
||||||
|
(tmp_path / "d.toml").write_text(row.format(family="not_a_family"))
|
||||||
|
(other,) = _load_catalog_file(tmp_path / "d.toml")
|
||||||
|
assert other.expected[0].family == "not_a_family"
|
||||||
|
|
||||||
|
|
||||||
|
def test_duplicate_bodies_across_pairs_are_reported_before_any_split() -> None:
|
||||||
|
from openultrasast.pairs import duplicate_groups
|
||||||
|
|
||||||
|
groups = duplicate_groups(select_vendored(load_pair_catalog(DEFAULT_CATALOG)))
|
||||||
|
assert all(len(names) > 1 for names in groups)
|
||||||
|
assert all(len(set(names)) == len(names) for names in groups)
|
||||||
|
report = Path("benchmarks/measurements/2026-09-06-corpus-hygiene.json")
|
||||||
|
assert report.is_file(), "the hygiene report is committed beside the numbers it explains"
|
||||||
|
payload = json.loads(report.read_text())
|
||||||
|
assert payload["duplicate_groups"] == [list(names) for names in groups]
|
||||||
|
assert set(payload["identical_twins"]) == IDENTICAL_TWINS
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_by_repository_keeps_declared_splits_and_reports_straddles() -> None:
|
||||||
|
from openultrasast.pairs import split_by_repository
|
||||||
|
|
||||||
|
cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG))
|
||||||
|
assigned, report = split_by_repository(cases)
|
||||||
|
declared = {case.name: case.split for case in cases if case.split_declared}
|
||||||
|
assert {case.name: case.split for case in assigned if case.name in declared} == declared # never re-splits a declared row
|
||||||
|
assert {name for name, _ in report.assigned} == {case.name for case in cases if not case.split_declared}
|
||||||
|
assert report.straddling, "repositories that sit in both splits are a corpus defect worth naming"
|
||||||
|
payload = json.loads(Path("benchmarks/measurements/2026-09-06-corpus-hygiene.json").read_text())
|
||||||
|
assert payload["straddling"] == [[repo, list(names)] for repo, names in report.straddling]
|
||||||
|
assert payload["undeclared_split"] == [list(item) for item in report.assigned]
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_by_repository_assigns_undeclared_rows_by_repository_and_date(tmp_path: Path) -> None:
|
||||||
|
from openultrasast.pairs import split_by_repository
|
||||||
|
|
||||||
|
def case(name: str, repo: str, date: str = "") -> PairCase:
|
||||||
|
(tmp_path / f"{name}.py").write_text(f"# {name}\n")
|
||||||
|
return PairCase(
|
||||||
|
name=name,
|
||||||
|
slice="sast",
|
||||||
|
language="python",
|
||||||
|
origin="test",
|
||||||
|
vuln_file=tmp_path / f"{name}.py",
|
||||||
|
fixed_file=tmp_path / f"{name}.py",
|
||||||
|
relpath="app.py",
|
||||||
|
expected=(),
|
||||||
|
min_recall=1.0,
|
||||||
|
fix_policy="silent",
|
||||||
|
repo=repo,
|
||||||
|
split="",
|
||||||
|
split_declared=False,
|
||||||
|
fix_date=date,
|
||||||
|
)
|
||||||
|
|
||||||
|
cases = [case("a", "one", "2024-01-01"), case("b", "one", "2026-01-01"), case("c", "two", "2025-01-01"), case("d", "two", "2025-06-01")]
|
||||||
|
assigned, report = split_by_repository(cases, holdout_fraction=0.5)
|
||||||
|
by_name = {item.name: item.split for item in assigned}
|
||||||
|
assert by_name["a"] == by_name["b"] and by_name["c"] == by_name["d"] # a repository never straddles
|
||||||
|
assert {by_name["a"], by_name["c"]} == {"train", "holdout"}
|
||||||
|
assert by_name["a"] == "train" # oldest repository trains, newest is held out
|
||||||
|
assert dict(report.assigned) == by_name and report.straddling == () and report.undated == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_pair_signals_can_be_filtered_to_one_split() -> None:
|
||||||
|
from openultrasast.pairs import PairOutcome, build_pair_signals
|
||||||
|
|
||||||
|
def outcome(name: str, split: str) -> PairOutcome:
|
||||||
|
return PairOutcome(
|
||||||
|
name=name,
|
||||||
|
slice="sast",
|
||||||
|
language="python",
|
||||||
|
origin="t",
|
||||||
|
detected_vuln=False,
|
||||||
|
silent_fix=True,
|
||||||
|
pair_correct=False,
|
||||||
|
vuln_matched=0,
|
||||||
|
vuln_expected=1,
|
||||||
|
vuln_findings=0,
|
||||||
|
fix_findings=0,
|
||||||
|
fix_leaks=0,
|
||||||
|
recall=0.0,
|
||||||
|
misses=("r:app.py:CWE-95",),
|
||||||
|
leaks=(),
|
||||||
|
split=split,
|
||||||
|
)
|
||||||
|
|
||||||
|
outcomes = [outcome("train-one", "train"), outcome("holdout-one", "holdout")]
|
||||||
|
assert {signal["pair"] for signal in build_pair_signals(outcomes)} == {"train-one", "holdout-one"}
|
||||||
|
assert {signal["pair"] for signal in build_pair_signals(outcomes, split="train")} == {"train-one"}
|
||||||
Reference in New Issue
Block a user