diff --git a/benchmarks/measurements/2026-09-06-corpus-hygiene.json b/benchmarks/measurements/2026-09-06-corpus-hygiene.json new file mode 100644 index 0000000..3d20d97 --- /dev/null +++ b/benchmarks/measurements/2026-09-06-corpus-hygiene.json @@ -0,0 +1,389 @@ +{ + "command": "python -c 'from openultrasast.pairs import ...; duplicate_groups / split_by_repository over the vendored catalog'", + "duplicate_groups": [ + [ + "openssl-cve-2023-0286", + "openssl-cve-2023-0286-general_name_cmp" + ], + [ + "openniw-jobs-0fc947", + "openniw-jobs-b74de3" + ], + [ + "curl-cve-2023-38545", + "curl-cve-2023-38545-do_socks5" + ], + [ + "openssl-cve-2021-3712", + "openssl-cve-2021-3712-append_ia5" + ], + [ + "openssl-cve-2020-1967", + "openssl-cve-2020-1967-tls1_check_sig_alg" + ], + [ + "openssl-cve-2022-3786", + "openssl-cve-2022-3786-ossl_punycode_decode" + ], + [ + "openssl-cve-2020-1971", + "openssl-cve-2020-1971-general_name_cmp" + ], + [ + "etude-lexical-markdown-editorconfig-c61a8b", + "popmark-editorconfig-e5ac40" + ], + [ + "anastasios-pw-session-14ec1a", + "astroclaw-pw-session-1553a8" + ], + [ + "openssl-cve-2022-3602", + "openssl-cve-2022-3602-ossl_punycode_decode" + ] + ], + "generated": "2026-09-06", + "identical_twins": [ + "dsvw-dsvw-do-get", + "python-insecure-app-main-try-hack-me", + "pythonssti-main-read-root", + "vampi-books-get-by-title", + "vulnpy-deserialization-do-pickle-load" + ], + "known_limit": [ + "owasp-java-hash" + ], + "note": "Repositories in `straddling` have pairs on both sides of the train/holdout split, so a learning path can be taught by one file of a repository and evaluated on another. Pairs in `duplicate_groups` share a vulnerable excerpt byte for byte after the provenance header is stripped. Both cap what any detector number can mean.", + "straddling": [ + [ + "Contrast-Security-OSS/DjanGoat", + [ + "djangoat-analytics-hits-by-ip", + "djangoat-views-reset-password", + "djangoat-views-user-view" + ] + ], + [ + "Contrast-Security-OSS/vulnpy", + [ + "vulnpy-cmdi-do-os-system", + "vulnpy-cmdi-do-subprocess-popen", + "vulnpy-deserialization-do-pickle-load" + ] + ], + [ + "LinuxUser255/Vulnerable_Python_Apps", + [ + "vulnerable-python-apps-bad-mvc-do-get", + "vulnerable-python-apps-dir-traversal-do-get" + ] + ], + [ + "OWASP-Benchmark/BenchmarkJava", + [ + "owasp-java-cmdi", + "owasp-java-hash", + "owasp-java-sqli" + ] + ], + [ + "OWASP-Benchmark/BenchmarkPython", + [ + "owasp-python-cmdi", + "owasp-python-codeinj", + "owasp-python-deser", + "owasp-python-hash", + "owasp-python-sqli" + ] + ], + [ + "anotherik/ThreatByte", + [ + "threatbyte-api-v1-delete", + "threatbyte-api-v1-get", + "threatbyte-routes-dashboard" + ] + ], + [ + "anxolerd/dvpwa", + [ + "dvpwa-app-init", + "dvpwa-middlewares-session-middleware", + "dvpwa-student-create" + ] + ], + [ + "arichardson/juliet-test-suite-c", + [ + "juliet-c-cwe121-strcpy", + "juliet-c-cwe134-printf" + ] + ], + [ + "bigfivefoods/supplieradvisor-mvp", + [ + "supplieradvisor-mvp-route-387bcf", + "supplieradvisor-mvp-route-a69e8f" + ] + ], + [ + "chromium/chromium", + [ + "chromium-cve-2019-5786", + "chromium-cve-2022-2162" + ] + ], + [ + "curl/curl", + [ + "curl-cve-2020-8177", + "curl-cve-2020-8231-curl_multi_add_handle", + "curl-cve-2020-8231-curl_open", + "curl-cve-2020-8284", + "curl-cve-2020-8284-config_init", + "curl-cve-2020-8286", + "curl-cve-2021-22876", + "curl-cve-2021-22890", + "curl-cve-2021-22898", + "curl-cve-2021-22901-ossl_connect_step1", + "curl-cve-2021-22924", + "curl-cve-2021-22924-nss_do_connect", + "curl-cve-2021-22924-servercert", + "curl-cve-2021-22925", + "curl-cve-2021-22945", + "curl-cve-2021-22946", + "curl-cve-2021-22946-imap_state_capability_re", + "curl-cve-2021-22946-pop3_state_capa_resp", + "curl-cve-2021-22947", + "curl-cve-2021-22947-imap_state_starttls_resp", + "curl-cve-2021-22947-pop3_state_starttls_resp", + "curl-cve-2021-22947-smtp_state_starttls_resp", + "curl-cve-2022-22576-conn_free", + "curl-cve-2022-27774", + "curl-cve-2022-27775", + "curl-cve-2022-27776", + "curl-cve-2022-27778", + "curl-cve-2022-27780", + "curl-cve-2022-27782-curl_free_primary_ssl_co", + "curl-cve-2022-27782-curl_init_userdefined", + "curl-cve-2022-27782-gtls_shutdown", + "curl-cve-2022-27782-nss_setup_connect", + "curl-cve-2022-27782-ossl_connect_step1", + "curl-cve-2022-30115", + "curl-cve-2022-32206", + "curl-cve-2022-32207", + "curl-cve-2022-32208", + "curl-cve-2023-38545", + "curl-cve-2023-38545-do_socks5", + "curl-cve-2026-9080" + ] + ], + [ + "dayflower/etude-lexical-markdown", + [ + "etude-lexical-markdown-editorconfig-c61a8b", + "etude-lexical-markdown-editorconfig-db4f54" + ] + ], + [ + "dolevf/Damn-Vulnerable-GraphQL-Application", + [ + "damn-vulnerable-graphql-application-views-mutate", + "damn-vulnerable-graphql-application-views-resolve-pastes", + "damn-vulnerable-graphql-application-views-resolve-system-dia" + ] + ], + [ + "erev0s/VAmPI", + [ + "vampi-books-get-by-title", + "vampi-user-model-get-user", + "vampi-users-update-password" + ] + ], + [ + "fportantier/vulpy", + [ + "vulpy-libuser-create", + "vulpy-libuser-login", + "vulpy-libuser-password-change" + ] + ], + [ + "manuelz120/extremely-vulnerable-flask-app", + [ + "extremely-vulnerable-flask-app-account-before-request", + "extremely-vulnerable-flask-app-account-search", + "extremely-vulnerable-flask-app-signup-validate-token" + ] + ], + [ + "naryal2580/vfapi", + [ + "vfapi-main-delete-user", + "vfapi-main-put-user", + "vfapi-main-sql-return-users-from-username" + ] + ], + [ + "openssl/openssl", + [ + "openssl-cve-2014-0160", + "openssl-cve-2020-1967", + "openssl-cve-2020-1967-tls1_check_sig_alg", + "openssl-cve-2020-1971", + "openssl-cve-2020-1971-general_name_cmp", + "openssl-cve-2020-25659", + "openssl-cve-2021-23840", + "openssl-cve-2021-23840-evp_encryptdecryptupdate", + "openssl-cve-2021-23841", + "openssl-cve-2021-3449", + "openssl-cve-2021-3449-def_generate_session_id", + "openssl-cve-2021-3449-init_sig_algs", + "openssl-cve-2021-3449-init_sig_algs_cert", + "openssl-cve-2021-3449-ssl_ctx_set_alpn_protos", + "openssl-cve-2021-3449-ssl_generate_master_secr", + "openssl-cve-2021-3449-tls_construct_ctos_early", + "openssl-cve-2021-3449-tls_handle_alpn", + "openssl-cve-2021-3449-tls_parse_ctos_renegotia", + "openssl-cve-2021-3449-tls_process_certificate_", + "openssl-cve-2021-3449-tls_process_finished", + "openssl-cve-2021-3450", + "openssl-cve-2021-3711", + "openssl-cve-2021-3711-pkey_sm2_decrypt", + "openssl-cve-2021-3711-sm2_asym_decrypt", + "openssl-cve-2021-3712", + "openssl-cve-2021-3712-append_ia5", + "openssl-cve-2021-3712-general_name_print", + "openssl-cve-2021-3712-i2r_pci", + "openssl-cve-2021-3712-ia5ncasecmp", + "openssl-cve-2021-3712-netscape_spki_print", + "openssl-cve-2021-3712-ossl_cmp_hdr_push1_freet", + "openssl-cve-2021-3712-print_qualifiers", + "openssl-cve-2021-4044", + "openssl-cve-2021-4044-tls_post_process_server_", + "openssl-cve-2022-0778", + "openssl-cve-2022-1292", + "openssl-cve-2022-1343", + "openssl-cve-2022-1434", + "openssl-cve-2022-2068-hash_dir", + "openssl-cve-2022-29242", + "openssl-cve-2022-3358", + "openssl-cve-2022-3358-evp_cipher_init_internal", + "openssl-cve-2022-3602", + "openssl-cve-2022-3602-ossl_punycode_decode", + "openssl-cve-2022-37454", + "openssl-cve-2022-3786", + "openssl-cve-2022-3786-ossl_punycode_decode", + "openssl-cve-2022-3996", + "openssl-cve-2022-4304", + "openssl-cve-2022-4304-bn_blinding_invert_ex", + "openssl-cve-2022-4304-rsa_blinding_invert", + "openssl-cve-2022-4304-rsa_ossl_private_decrypt", + "openssl-cve-2023-0216", + "openssl-cve-2023-0217", + "openssl-cve-2023-0286", + "openssl-cve-2023-0286-general_name_cmp", + "openssl-cve-2023-0401", + "openssl-cve-2023-0464", + "openssl-cve-2023-0464-tree_init", + "openssl-cve-2023-0465", + "openssl-cve-2023-2650", + "openssl-cve-2023-2975", + "openssl-cve-2023-3446", + "openssl-cve-2023-3817-dh_check", + "openssl-cve-2023-5363", + "openssl-cve-2024-0727", + "openssl-cve-2024-0727-smime_write_pkcs7", + "openssl-cve-2024-12797", + "openssl-cve-2024-13176-ec_field_inverse_mod_ord", + "openssl-cve-2024-2511", + "openssl-cve-2024-2511-tls_construct_server_hel", + "openssl-cve-2024-4603", + "openssl-cve-2024-4741", + "openssl-cve-2024-4741-ssl3_release_read_buffer", + "openssl-cve-2024-4741-ssl_free_buffers", + "openssl-cve-2024-4741-tls_free_buffers", + "openssl-cve-2024-4741-tls_release_read_buffer", + "openssl-cve-2024-5535", + "openssl-cve-2024-5535-alpn_select_cb", + "openssl-cve-2024-5535-ssl_select_next_proto", + "openssl-cve-2024-5535-tls_construct_stoc_next_", + "openssl-cve-2024-5535-tls_parse_stoc_npn", + "openssl-cve-2024-6119", + "openssl-cve-2025-11187", + "openssl-cve-2025-15467", + "openssl-cve-2025-15469", + "openssl-cve-2025-66199", + "openssl-cve-2025-66199-tls13_process_compressed", + "openssl-cve-2025-68160", + "openssl-cve-2025-69418", + "openssl-cve-2025-69419", + "openssl-cve-2025-69421", + "openssl-cve-2025-9230", + "openssl-cve-2025-9232", + "openssl-cve-2026-14456-expect_quic_for_value", + "openssl-cve-2026-14456-port_default_packet_hand", + "openssl-cve-2026-14457", + "openssl-cve-2026-18798", + "openssl-cve-2026-2673", + "openssl-cve-2026-28388", + "openssl-cve-2026-28389", + "openssl-cve-2026-28389-dh_cms_set_shared_info", + "openssl-cve-2026-28389-ecdh_cms_set_shared_info", + "openssl-cve-2026-31789", + "openssl-cve-2026-31790", + "openssl-cve-2026-31790-rsasve_generate", + "openssl-cve-2026-34181", + "openssl-cve-2026-35188", + "openssl-cve-2026-42764", + "openssl-cve-2026-42765", + "openssl-cve-2026-42766", + "openssl-cve-2026-42768", + "openssl-cve-2026-42768-pkcs7_decrypt_rinfo", + "openssl-cve-2026-42769", + "openssl-cve-2026-42771", + "openssl-cve-2026-45445", + "openssl-cve-2026-45446", + "openssl-cve-2026-45446-aes_siv_set_ctx_params", + "openssl-cve-2026-45447", + "openssl-cve-2026-54874", + "openssl-cve-2026-54874-dtls1_process_record", + "openssl-cve-2026-54874-dtls_rlayer_buffer_recor", + "openssl-cve-2026-63072", + "openssl-cve-2026-63072-cms_kek_cipher", + "openssl-cve-2026-63073", + "openssl-cve-2026-63074", + "openssl-cve-2026-63075-txp_pkt_commit", + "openssl-cve-2026-7383", + "openssl-cve-2026-75803", + "openssl-cve-2026-75803-aes_gcm_siv_finish", + "openssl-cve-2026-75803-chacha20_poly1305_cipher", + "openssl-cve-2026-9076" + ] + ], + [ + "sgabe/DSVPWA", + [ + "dsvpwa-attacks-run", + "dsvpwa-dsvpwa-main", + "dsvpwa-handlers-do-bdr" + ] + ] + ], + "undeclared_split": [ + [ + "local-python-vulnerable", + "holdout" + ], + [ + "local-javascript-vulnerable", + "holdout" + ], + [ + "local-cpp-buffer", + "train" + ] + ], + "vendored_pairs": 277 +} diff --git a/src/openultrasast/benchmark.py b/src/openultrasast/benchmark.py index b3714ce..528a4e6 100644 --- a/src/openultrasast/benchmark.py +++ b/src/openultrasast/benchmark.py @@ -28,6 +28,7 @@ class ExpectedFinding: sink: str | None = None mechanism: str | None = None obligation: str | None = None # additive (authorization-obligations, Req 7.2): an OPERATION_KIND the labeled function owes + family: str | None = None # additive (learning-harness, Req 4): an opaque family id; the classifier owns the taxonomy @dataclass(frozen=True) diff --git a/src/openultrasast/pairs.py b/src/openultrasast/pairs.py index 1046738..b8a069e 100644 --- a/src/openultrasast/pairs.py +++ b/src/openultrasast/pairs.py @@ -13,7 +13,9 @@ evaluated but excluded from the achievable numbers. from __future__ import annotations +import hashlib import importlib.util +import math import os import shutil import tempfile @@ -126,6 +128,9 @@ class PairCase: review_tier: str = "advisory" reviewer: str = "" vendored: bool = True + unscorable: str | None = None # computed: identical_twin, known_limit: (learning-harness Req 4.1) + split_declared: bool = True # False when the catalog row named no split, so the split helper may assign one + fix_date: str = "" # ISO date of the fixing commit; the catalog generator fills it, the split helper orders by it recipe: tuple[tuple[str, object], ...] = () # pointer pairs carry their harvest recipe instead of excerpts @@ -276,6 +281,9 @@ def _load_catalog_file(path: Path) -> tuple[PairCase, ...]: provenance=provenance, split=split, known_limit=str(known_limit) if known_limit else None, + unscorable=_unscorable_reason(known_limit, vuln_file, fixed_file, vendored=vendored), + split_declared="split" in item, + fix_date=str(item.get("fix_date", "")), review_tier=review_tier, reviewer=reviewer, vendored=vendored, @@ -459,12 +467,17 @@ def evaluate_catalog( ) -def build_pair_signals(outcomes: Sequence[PairOutcome]) -> list[dict[str, object]]: - """Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name, profile, and mechanisms.""" +def build_pair_signals(outcomes: Sequence[PairOutcome], *, split: str | None = None) -> list[dict[str, object]]: + """Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name, profile, and mechanisms. + + ``split`` restricts the signals to one split, so a learning path teaches from the train rows only (Req 5.1). + """ signals: list[dict[str, object]] = [] for outcome in outcomes: if outcome.known_limit is not None: continue # unachievable by construction; the loop must not chase it + if split is not None and outcome.split != split: + continue common: dict[str, object] = { "pair": outcome.name, "profile": outcome.provenance, @@ -480,6 +493,87 @@ def build_pair_signals(outcomes: Sequence[PairOutcome]) -> list[dict[str, object return sorted(signals, key=lambda item: (str(item.get("signal")), str(item.get("rule_id")), str(item.get("pair")))) +@dataclass(frozen=True) +class SplitReport: + """What ``split_by_repository`` had to do and what it found (learning-harness Req 4.4).""" + + assigned: tuple[tuple[str, str], ...] = () # (pair, split) for rows whose catalog row named none + straddling: tuple[tuple[str, tuple[str, ...]], ...] = () # repository -> pairs, when one repository sits in both splits + undated: int = 0 # assigned rows with no fix date, ordered by name instead + + +@lru_cache(maxsize=4096) +def _body_digest(path: Path) -> str | None: + """Hash of an excerpt with its provenance header and trailing whitespace removed; None when unreadable or empty.""" + try: + text = path.read_text(errors="ignore") + except OSError: + return None + lines = text.splitlines() + index = 0 + while index < len(lines) and (not lines[index].strip() or lines[index].lstrip().startswith(("#", "//", "/*", "*"))): + index += 1 + body = "\n".join(line.rstrip() for line in lines[index:]).strip() + return hashlib.sha256(body.encode()).hexdigest() if body else None + + +def _unscorable_reason(known_limit: object, vuln_file: Path, fixed_file: Path, *, vendored: bool) -> str | None: + """Why this pair cannot be scored, or None. A declared known limit wins over anything computed.""" + if known_limit: + return f"known_limit:{known_limit}" + if not vendored: + return None + digest = _body_digest(vuln_file) + return "identical_twin" if digest is not None and digest == _body_digest(fixed_file) else None + + +def duplicate_groups(cases: Sequence[PairCase]) -> tuple[tuple[str, ...], ...]: + """Pairs whose vulnerable excerpts are the same code, reported before any split is applied (Req 4.3).""" + by_digest: dict[str, list[str]] = {} + for case in cases: + digest = _body_digest(case.vuln_file) if case.vendored else None + if digest is not None: + by_digest.setdefault(digest, []).append(case.name) + return tuple(tuple(sorted(names)) for _, names in sorted(by_digest.items()) if len(names) > 1) + + +def split_by_repository(cases: Sequence[PairCase], *, holdout_fraction: float = 0.5) -> tuple[tuple[PairCase, ...], SplitReport]: + """Keep every declared split, assign the rest by repository in date order, and report repositories that straddle. + + A repository never straddles by our doing: rows that named no split inherit their repository's declared split when + there is one, and otherwise the whole repository lands on one side, oldest first (Req 4.4). + """ + groups: dict[str, list[PairCase]] = {} + for case in cases: + groups.setdefault(case.repo or f"{case.slice}:{case.name}", []).append(case) + straddling = tuple( + (repo, tuple(sorted(case.name for case in members))) + for repo, members in sorted(groups.items()) + if len({case.split for case in members if case.split_declared}) > 1 + ) + open_repos = [repo for repo, members in groups.items() if any(not case.split_declared for case in members)] + inherited = {repo: next((case.split for case in groups[repo] if case.split_declared), None) for repo in open_repos} + orderable = sorted( + (repo for repo in open_repos if inherited[repo] is None), + key=lambda repo: (min((case.fix_date for case in groups[repo] if case.fix_date), default="9999-99-99"), repo), + ) + holdout_count = math.ceil(len(orderable) * holdout_fraction) if orderable else 0 + holdout = set(orderable[len(orderable) - holdout_count :]) if holdout_count else set() + assigned: list[tuple[str, str]] = [] + undated = 0 + out: list[PairCase] = [] + for case in cases: + repo = case.repo or f"{case.slice}:{case.name}" + if case.split_declared: + out.append(case) + continue + split = inherited.get(repo) or ("holdout" if repo in holdout else "train") + undated += 0 if case.fix_date else 1 + assigned.append((case.name, split)) + out.append(replace(case, split=split, split_declared=True)) + return tuple(out), SplitReport(assigned=tuple(assigned), straddling=straddling, undated=undated) + + def select_slice(cases: Sequence[PairCase], slice_name: str | None) -> tuple[PairCase, ...]: if not slice_name or slice_name == "all": return tuple(cases) @@ -558,6 +652,7 @@ def _parse_expected(item: dict[str, object], name: str, mechanisms: frozenset[st raise CatalogError(f"pair {name}: expected row is missing mechanism") if mechanism not in mechanisms: raise CatalogError(f"pair {name}: unknown mechanism {mechanism!r}; see benchmarks/pairs/mechanisms.toml") + family = item.get("family") obligation = item.get("obligation") if obligation is not None and obligation not in OPERATION_KINDS: raise CatalogError(f"pair {name}: unknown obligation {obligation!r}; expected one of {sorted(OPERATION_KINDS)}") @@ -572,6 +667,7 @@ def _parse_expected(item: dict[str, object], name: str, mechanisms: frozenset[st sink=str(item["sink"]) if "sink" in item else None, mechanism=mechanism, obligation=str(obligation) if obligation is not None else None, + family=str(family) if family is not None else None, ) diff --git a/tests/test_pair_hygiene.py b/tests/test_pair_hygiene.py new file mode 100644 index 0000000..7908a71 --- /dev/null +++ b/tests/test_pair_hygiene.py @@ -0,0 +1,149 @@ +"""learning-harness task 1.3: additive corpus fields and hygiene at load (offline).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored + +IDENTICAL_TWINS = { + "dsvw-dsvw-do-get", + "python-insecure-app-main-try-hack-me", + "pythonssti-main-read-root", + "vampi-books-get-by-title", + "vulnpy-deserialization-do-pickle-load", +} +VENDORED_PER_SLICE = {"local": 3, "github": 6, "sast": 11, "vibe-py": 35, "vfc-js": 17, "vfc": 176, "agent-vfc": 29} + + +def test_identical_twins_are_unscorable_and_no_row_is_dropped() -> None: + cases = load_pair_catalog(DEFAULT_CATALOG) + per_slice: dict[str, int] = {} + for case in select_vendored(cases): + per_slice[case.slice] = per_slice.get(case.slice, 0) + 1 + assert per_slice == VENDORED_PER_SLICE # nothing dropped by the new accounting + twins = {case.name for case in cases if case.unscorable == "identical_twin"} + assert twins == IDENTICAL_TWINS + assert all(case.unscorable is None for case in cases if case.name not in IDENTICAL_TWINS and not case.known_limit) + + +def test_a_declared_known_limit_wins_over_a_computed_reason(tmp_path: Path) -> None: + from openultrasast.pairs import _load_catalog_file + + (tmp_path / "v.py").write_text("# provenance: x\n\ndef f():\n return 1\n") + (tmp_path / "f.py").write_text("# provenance: y\n\ndef f():\n return 1\n") + head = ( + '[[pair]]\nname = "t"\nslice = "sast"\nvuln = "v.py"\nfixed = "f.py"\nrelpath = "app.py"\n{extra}\n' + '[[pair.expected]]\ncwe = "CWE-95"\nclass = "x"\npath = "app.py"\nsink = "eval"\nmechanism = "other"\n' + ) + (tmp_path / "plain.toml").write_text(head.format(extra="")) + (case,) = _load_catalog_file(tmp_path / "plain.toml") + assert case.unscorable == "identical_twin" # the provenance headers differ, the bodies do not + (tmp_path / "limited.toml").write_text(head.format(extra='known_limit = "java hash"')) + (limited,) = _load_catalog_file(tmp_path / "limited.toml") + assert limited.unscorable == "known_limit:java hash" + + +def test_expected_rows_carry_an_opaque_family_label(tmp_path: Path) -> None: + from openultrasast.pairs import _load_catalog_file + + (tmp_path / "v.py").write_text("x = 1\n") + (tmp_path / "f.py").write_text("x = 2\n") + row = ( + '[[pair]]\nname = "t"\nslice = "sast"\nvuln = "v.py"\nfixed = "f.py"\nrelpath = "app.py"\n\n' + '[[pair.expected]]\ncwe = "CWE-95"\nclass = "x"\npath = "app.py"\nsink = "eval"\nmechanism = "other"\nfamily = "{family}"\n' + ) + (tmp_path / "c.toml").write_text(row.format(family="injection")) + (case,) = _load_catalog_file(tmp_path / "c.toml") + assert case.expected[0].family == "injection" + # the corpus stores the label; the classifier owns the taxonomy, so a fabricated id is not rejected here + (tmp_path / "d.toml").write_text(row.format(family="not_a_family")) + (other,) = _load_catalog_file(tmp_path / "d.toml") + assert other.expected[0].family == "not_a_family" + + +def test_duplicate_bodies_across_pairs_are_reported_before_any_split() -> None: + from openultrasast.pairs import duplicate_groups + + groups = duplicate_groups(select_vendored(load_pair_catalog(DEFAULT_CATALOG))) + assert all(len(names) > 1 for names in groups) + assert all(len(set(names)) == len(names) for names in groups) + report = Path("benchmarks/measurements/2026-09-06-corpus-hygiene.json") + assert report.is_file(), "the hygiene report is committed beside the numbers it explains" + payload = json.loads(report.read_text()) + assert payload["duplicate_groups"] == [list(names) for names in groups] + assert set(payload["identical_twins"]) == IDENTICAL_TWINS + + +def test_split_by_repository_keeps_declared_splits_and_reports_straddles() -> None: + from openultrasast.pairs import split_by_repository + + cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG)) + assigned, report = split_by_repository(cases) + declared = {case.name: case.split for case in cases if case.split_declared} + assert {case.name: case.split for case in assigned if case.name in declared} == declared # never re-splits a declared row + assert {name for name, _ in report.assigned} == {case.name for case in cases if not case.split_declared} + assert report.straddling, "repositories that sit in both splits are a corpus defect worth naming" + payload = json.loads(Path("benchmarks/measurements/2026-09-06-corpus-hygiene.json").read_text()) + assert payload["straddling"] == [[repo, list(names)] for repo, names in report.straddling] + assert payload["undeclared_split"] == [list(item) for item in report.assigned] + + +def test_split_by_repository_assigns_undeclared_rows_by_repository_and_date(tmp_path: Path) -> None: + from openultrasast.pairs import split_by_repository + + def case(name: str, repo: str, date: str = "") -> PairCase: + (tmp_path / f"{name}.py").write_text(f"# {name}\n") + return PairCase( + name=name, + slice="sast", + language="python", + origin="test", + vuln_file=tmp_path / f"{name}.py", + fixed_file=tmp_path / f"{name}.py", + relpath="app.py", + expected=(), + min_recall=1.0, + fix_policy="silent", + repo=repo, + split="", + split_declared=False, + fix_date=date, + ) + + cases = [case("a", "one", "2024-01-01"), case("b", "one", "2026-01-01"), case("c", "two", "2025-01-01"), case("d", "two", "2025-06-01")] + assigned, report = split_by_repository(cases, holdout_fraction=0.5) + by_name = {item.name: item.split for item in assigned} + assert by_name["a"] == by_name["b"] and by_name["c"] == by_name["d"] # a repository never straddles + assert {by_name["a"], by_name["c"]} == {"train", "holdout"} + assert by_name["a"] == "train" # oldest repository trains, newest is held out + assert dict(report.assigned) == by_name and report.straddling == () and report.undated == 0 + + +def test_pair_signals_can_be_filtered_to_one_split() -> None: + from openultrasast.pairs import PairOutcome, build_pair_signals + + def outcome(name: str, split: str) -> PairOutcome: + return PairOutcome( + name=name, + slice="sast", + language="python", + origin="t", + detected_vuln=False, + silent_fix=True, + pair_correct=False, + vuln_matched=0, + vuln_expected=1, + vuln_findings=0, + fix_findings=0, + fix_leaks=0, + recall=0.0, + misses=("r:app.py:CWE-95",), + leaks=(), + split=split, + ) + + outcomes = [outcome("train-one", "train"), outcome("holdout-one", "holdout")] + assert {signal["pair"] for signal in build_pair_signals(outcomes)} == {"train-one", "holdout-one"} + assert {signal["pair"] for signal in build_pair_signals(outcomes, split="train")} == {"train-one"}