From d1dc0d113113e17c00ff60e3e1fc4a59097768ec Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Thu, 24 Sep 2026 16:43:32 +0200 Subject: [PATCH] spec(pre-push-safety-net): freeze the independent population v1 (15.1) Every reserved case verified with git and file reads only -- nothing scanned: pins present and ordered, the sink present at the vulnerable pin, the fix touching the sink (or LearnPress's declared source-side fix site), and the sink file hashed at both pins (benchmarks/independent/freeze.py, record freeze-v1.json). Each case now carries a benign control from its own history, reviewed by diff. The mechanical keyword filter was not enough: it proposed an access restriction, a filter check, API-token scopes, a login change and a TLS-verification option, all rejected; Ultimate Member and alerta got hand-picked controls (a WPCS reformat; a dependency change with both sides still vulnerable). The guard test requires a benign pair per case once frozen. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .kiro/specs/pre-push-safety-net/tasks.md | 1 + benchmarks/independent/freeze-v1.json | 334 ++++++++++++++++++++++ benchmarks/independent/freeze.py | 171 +++++++++++ benchmarks/independent/population-v1.toml | 19 +- tests/test_independent_population.py | 7 + 5 files changed, 530 insertions(+), 2 deletions(-) create mode 100644 benchmarks/independent/freeze-v1.json create mode 100644 benchmarks/independent/freeze.py diff --git a/.kiro/specs/pre-push-safety-net/tasks.md b/.kiro/specs/pre-push-safety-net/tasks.md index d8a186b..ff3d6f4 100644 --- a/.kiro/specs/pre-push-safety-net/tasks.md +++ b/.kiro/specs/pre-push-safety-net/tasks.md @@ -598,6 +598,7 @@ Preserve all completed task evidence below. - **The population MUST hold a multi-language repository.** Task 13.4 found that a second language in a repository cost every question its completion, and so cost every finding its admission — on the declared v0.1 target, which is PHP plus admin JavaScript. Neither subject that passes today could reveal it, because both hold exactly one language. A population of single-language repositories would qualify a capability that fails on the product's own target. - Done when a frozen population exists with its provenance and review recorded, at least one member holds more than one language, and nothing in it has been scanned. - _Reserved 2026-09-24, not yet frozen._ Eleven cases selected by the maintainer from fifteen researched by advisory and fix diff alone (no clone, no scan), none among the 262 repositories this project's benchmarks reference, plus the earlier Ghost reservation: 4 PHP with JavaScript (Ultimate Member and LearnPress WordPress plugins, phpMyFAQ, YesWiki; all SQL injection), 4 Node/TypeScript (Signal K SSRF, Saltcorn SQLi, mongo-express code injection, Mockoon permissive CORS), 3 Python (alerta SQLi, wger open redirect, OpenCVE SSRF). Record: `benchmarks/independent/population-v1.toml`; research and rejected alternates: `research-2026-09-24.md`. A guard test fails if any reserved repository is referenced elsewhere in the tree. Remaining to FREEZE: per case, verify the pins and the sink by reading source, and choose a benign change from the same history without scanning; the OpenCVE licence must be checked before any result is published. + - _Frozen 2026-09-24._ `benchmarks/independent/freeze.py` verified every case with git and file reads only -- no analyzer, query or tooling: both pins present and ordered, the sink text present at the vulnerable pin (LearnPress's fix is source-side and recorded as `fix_site`; Mockoon's sink is recorded as `sink_evidence`), the fix touching the sink or its declared fix site, and SHA-256 of the sink file at both pins. Benign controls were proposed mechanically and REVIEWED by diff: the keyword filter alone let through an access-restriction, a filter check, API-token scopes, a login change and a TLS-verification option, all rejected; Ultimate Member's and alerta's controls were replaced by hand (a coding-standards reformat; a dependency-plumbing change with both sides still vulnerable). Record: `freeze-v1.json`. The guard test now requires a benign pair for every case of a frozen population. Nothing in it has been scanned; the first scan is the evaluation. - _Boundary: Evaluation harness, corpus provenance_ - _Requirements: 8.1, 8.2_ - [ ] 15.2 Declare the v1 advisory scope as the families whose claims can be admitted diff --git a/benchmarks/independent/freeze-v1.json b/benchmarks/independent/freeze-v1.json new file mode 100644 index 0000000..5611438 --- /dev/null +++ b/benchmarks/independent/freeze-v1.json @@ -0,0 +1,334 @@ +{ + "schema_version": 1, + "population": "population-v1.toml", + "cases": [ + { + "id": "wp-ultimate-member-sqli", + "repo": "https://github.com/ultimatemember/ultimatemember", + "vulnerable": "a49b3061ded1630e4f6860b4ebece5ccf0a493d2", + "fixed": "476de92f4c9aaa56c3e2f5527b242ce0476f4655", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "includes/core/class-member-directory-meta.php", + "sink_functions": [ + "ajax_get_members" + ], + "sink_sha256_vulnerable": "fc6f64e3570f69637a05d747c24bc6fd5e5b350e0a9b1639ef1114d76177c719", + "sink_functions_missing": [], + "sink_sha256_fixed": "2f7a6a913a1ac0c0bed728c8b53d4a32435bafe4f50756f7d8fe30effc789fd7", + "fix_changed_files": 1, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "13151ef558eb6998720cb1ee5f78b5f17535c16c", + "tip": "9130a3226ffec45569a2872ffac338200bd1bfcb", + "subject": "- wpcs;", + "manual": true + }, + "benign_review": "replaced: the proposal ea316d04 changed account-status/deactivation logic (authorization-adjacent); 9130a322 is a WPCS reformat of class-dependencies.php" + }, + { + "id": "wp-learnpress-sqli", + "repo": "https://github.com/LearnPress/learnpress", + "vulnerable": "73b3cbceef8669056e9da5e2f2c78944eb139af5", + "fixed": "aaf42fe0c6bf05dcf2ff125750b7bd836e76fa3c", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "inc/Databases/class-lp-db.php", + "sink_functions": [ + "execute" + ], + "sink_sha256_vulnerable": "5a1e22fa525b56771676e6dd7562c796146e52b96e4e0c11471bf2f0d1d329ce", + "sink_functions_missing": [], + "sink_sha256_fixed": "5a1e22fa525b56771676e6dd7562c796146e52b96e4e0c11471bf2f0d1d329ce", + "fix_changed_files": 4, + "fix_touches_sink_file": false, + "fix_touches_fix_site": true, + "benign_proposal": { + "base": "aaf42fe0c6bf05dcf2ff125750b7bd836e76fa3c", + "tip": "179ed7760e6ae5fa46fbe9c5f1324278a42ece04", + "subject": "= 4.2.7.1 = ~ Added: lpOnElementReady function js", + "files": [ + "assets/src/js/loadAJAX.js", + "assets/src/js/utils.js", + "inc/class-lp-assets.php" + ], + "lines": 50 + }, + "benign_review": "accepted: JS utility and asset registration" + }, + { + "id": "phpmyfaq-captcha-ua-sqli", + "repo": "https://github.com/thorsten/phpMyFAQ", + "vulnerable": "11583abcf4787543d00c25d6c7926af6f509faf0", + "fixed": "545bdffb11244a4741cd29ac909a849c9a6a2e53", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php", + "sink_functions": [ + "garbageCollector", + "saveCaptcha" + ], + "sink_sha256_vulnerable": "87e468ef69ea87d20d2bafe4ec3a2b5a796fa96faaaac114334e98e72f85a5fb", + "sink_functions_missing": [], + "sink_sha256_fixed": "37c38d1fc7a8ee0dc3a79b69f3fdfb2c9e0f3b90eef078c027e8884b2975dfbc", + "fix_changed_files": 2, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "bf781788ab6b5c2967a09d66216d7580778a2b5a", + "tip": "a4601ae5f01761dc0bb91b8c0c3901be91987ba3", + "subject": "build: updated version strings to 4.2", + "files": [ + "CHANGELOG.md", + "docker-compose.yml", + "docs/update.md", + "mkdocs.yml", + "package.json", + "phpmyfaq/src/phpMyFAQ/Setup/AbstractSetup.php", + "phpmyfaq/src/phpMyFAQ/Setup/Installer.php", + "phpmyfaq/src/phpMyFAQ/Setup/Update.php", + "phpmyfaq/src/phpMyFAQ/Setup/UpdateRunner.php", + "phpmyfaq/src/phpMyFAQ/System.php", + "scripts/version.sh", + "tests/phpMyFAQ/SystemTest.php" + ], + "lines": 157 + }, + "benign_review": "accepted: version strings" + }, + { + "id": "yeswiki-bazar-form-sqli", + "repo": "https://github.com/YesWiki/yeswiki", + "vulnerable": "6f222915df36fca628fc83325cb0feed8dc2aca0", + "fixed": "fe7244bffcbeffd42777496ebc2a6f6118cc2388", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "tools/bazar/services/FormManager.php", + "sink_functions": [ + "create" + ], + "sink_sha256_vulnerable": "2051be959575f429ad51367ea6a2e592bc3eb0e86756e93cca3715a942084d3c", + "sink_functions_missing": [], + "sink_sha256_fixed": "f71d844466f0466d8b3e5963670c29b6730f5e675de985b466137a03ad3e1ef9", + "fix_changed_files": 2, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "03f27fdde5f89f90de76cd3f2160cff82e7bffac", + "tip": "70d240c0b43f7e7b0bac55fc0431fb4c2d88b49c", + "subject": "fix(contact): send shared page by mail", + "files": [ + "tools/contact/libs/contact.functions.php" + ], + "lines": 7 + }, + "benign_review": "accepted: mail receiver string-to-array normalisation and comments" + }, + { + "id": "signalk-remote-conn-ssrf", + "repo": "https://github.com/SignalK/signalk-server", + "vulnerable": "1c479cc0d2d26d14ed74416cff36fa7336c7bf9e", + "fixed": "2816b0dda1bdde893d85c969a9c5950e7825eee7", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "src/serverroutes.ts", + "sink_functions": [ + "makeRemoteRequest" + ], + "sink_sha256_vulnerable": "10b2224bfdc0308bf1b7f92ee2f37aaabcfdc6578cda59d245e3e97ddea8185f", + "sink_functions_missing": [], + "sink_sha256_fixed": "9aeb864d90df914181683ada2bd49a8e4a86cda4a1a2caa160025225001801bd", + "fix_changed_files": 5, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "8af993d45c898edfe28f820f1726b6fc33b27cad", + "tip": "07416b79601c5d901f671f70a94bd57bed90bad8", + "subject": "fix: allow multiple deltaHandler next calls (#2818)", + "files": [ + "src/deltachain.ts", + "test/deltachain.ts" + ], + "lines": 61 + }, + "benign_review": "accepted: delta handler chaining fix" + }, + { + "id": "saltcorn-sync-sqli", + "repo": "https://github.com/saltcorn/saltcorn", + "vulnerable": "e7e09254a3c69dc6582eda9e5b5769935db108af", + "fixed": "2b6c8a2eab88955ea6b12b1b4e5c2e613faefef5", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "packages/server/routes/sync.js", + "sink_functions": [ + "getSyncRows", + "getDelRows" + ], + "sink_sha256_vulnerable": "b69b79171574ba7d8ca4b02d5c0ffcadeb0f885faab016d8066ebf753d839edf", + "sink_functions_missing": [], + "sink_sha256_fixed": "5b5d34758caca6601e327953191ca30475193e389bea542549a91f82af2bf8dc", + "fix_changed_files": 5, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "5c619403c40447bf56ce662570d3a3ffbf781019", + "tip": "df114765fd36d280395b2129f1703fa98c712d06", + "subject": "updated at in trigger.findAllWithTableName", + "files": [ + "packages/saltcorn-data/models/trigger.ts" + ], + "lines": 2 + }, + "benign_review": "accepted: adds a constant column to a SQL select -- a query edited without any input, a useful negative control" + }, + { + "id": "mongo-express-bson-rce", + "repo": "https://github.com/mongo-express/mongo-express", + "vulnerable": "ea02b364d43f179f191fc91fb9962efdb0843a8d", + "fixed": "d8c9bda46a204ecba1d35558452685cd0674e6f2", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "lib/bson.js", + "sink_functions": [ + "toBSON" + ], + "sink_sha256_vulnerable": "09032fcec4e4b0782f94bb69bd946afeee9794991daa4b6a9d3c7aa0b1b5531c", + "sink_functions_missing": [], + "sink_sha256_fixed": "a9fa97927fbbb8460902813beb8b99e7d23cda6be7f3761420259aaaf9268640", + "fix_changed_files": 6, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "9550ae2529258b47a2b637d8e64e69a0d4d38ad8", + "tip": "5b9b34f8553d004ceb7b666bbf566b406a2a3153", + "subject": "Merge pull request #538 from damien-malescot/master", + "files": [ + "lib/routes/document.js", + "lib/views/document.html" + ], + "lines": 7 + }, + "benign_review": "accepted: UUID document-delete fix" + }, + { + "id": "mockoon-admin-api-cors", + "repo": "https://github.com/mockoon/mockoon", + "vulnerable": "b42bdfb7f82e83f0e81bea8e6fe41adf5ec82585", + "fixed": "c420b5a56918475b8663977b51e5f986e45b3299", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "packages/commons-server/src/libs/server/admin-api.ts", + "sink_functions": [ + "middleware" + ], + "sink_sha256_vulnerable": "1beb42bf35daa8c7291a1f75976a7012cdd29411d909218a27544ec06eae92f9", + "sink_evidence_found": true, + "sink_sha256_fixed": "81866aec8e3a67f3aef78c40980193dab841934102fce53ca4f53c73c2ed6eab", + "fix_changed_files": 34, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "de69f5e30f89e17918855d147f4e8c055d5b2ba9", + "tip": "895961070b55a14d290ad50232ab9c6d800c3cad", + "subject": "Fix serverless lib returning a 404 when no body is present (#2274)", + "files": [ + "packages/commons-server/src/libs/server/server.ts", + "packages/commons-server/test/specs/server/server-body-parsing.test.ts" + ], + "lines": 40 + }, + "benign_review": "accepted: serverless 404 on empty body" + }, + { + "id": "alerta-query-sqli", + "repo": "https://github.com/alerta/alerta", + "vulnerable": "71c098d8bb8091e76edaefeef4662eb77c753ce7", + "fixed": "fdd52cd1abad8d02d1dfb8ecdcdbb43b6af3b883", + "problems": [ + "no benign change found automatically; choose one by hand" + ], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "alerta/database/backends/postgres/queryparser.py", + "sink_functions": [ + "__repr__" + ], + "sink_sha256_vulnerable": "40884d7e83cc38db30a03b11f9686c8cb4b629c2de74191175516cb4c4f1ad3e", + "sink_functions_missing": [], + "sink_sha256_fixed": "d9c13317bb36208bbe83bd305bed1199a9e7a586659ca8256479f6158b9d0ce6", + "fix_changed_files": 3, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "b7111d23ae9aefb5064ef107976f62e3836217b6", + "tip": "71c098d8bb8091e76edaefeef4662eb77c753ce7", + "subject": "fix(deps): replace pkg_resources with importlib.metadata (#2041)", + "manual": true + }, + "benign_review": "replaced: no neutral commit after the fix (proposals were login, TLS-verification and filter changes); 71c098d8 replaces pkg_resources with importlib.metadata, both sides still vulnerable" + }, + { + "id": "wger-next-open-redirect", + "repo": "https://github.com/wger-project/wger", + "vulnerable": "d30e85b23fb1307ac6579e9e3ae393c50470cea7", + "fixed": "0ba371bbb50cd5668820f240995405e1b2483384", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "wger/core/views/user.py", + "sink_functions": [ + "trainer_login", + "use_browser_language" + ], + "sink_sha256_vulnerable": "185582190d001b1670738dde2264dac5e6fef2253cc8595dc862c74a270e906f", + "sink_functions_missing": [ + "use_browser_language" + ], + "sink_sha256_fixed": "c0952e4b1990081906cb90a597a83cabafd1a3f8e2d124c4c9b1e2ab80756534", + "fix_changed_files": 4, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "47d646c24178783eb6787ecf26e79dc8f4a394c6", + "tip": "c1144533bf9bca487ff905d925b567151d89c1aa", + "subject": "Formatting", + "files": [ + "wger/core/api/serializers.py" + ], + "lines": 5 + }, + "benign_review": "accepted: formatting" + }, + { + "id": "opencve-notification-ssrf", + "repo": "https://github.com/opencve/opencve", + "vulnerable": "928bbadbb1672051bcfc2a6289a9e6a8f029c106", + "fixed": "0af7f894debc95c67fa53e7ba5a6cdecb10c34f0", + "problems": [], + "vulnerable_is_ancestor_of_fixed": true, + "sink_file": "web/projects/notifications.py", + "sink_functions": [ + "_http_post_json" + ], + "sink_sha256_vulnerable": "392b560fdbfe29f0d56661a5a1e74afcca426ce6658eb96d1c0ff12c8cacb1ad", + "sink_functions_missing": [], + "sink_sha256_fixed": "e8cc8b410e1e43bbf5356066a800c61a32f7d13f03c449512c761dd67553e933", + "fix_changed_files": 8, + "fix_touches_sink_file": true, + "fix_touches_fix_site": false, + "benign_proposal": { + "base": "0c8101082ba20628b28fcec9c4611745d32202c2", + "tip": "965dc348e225308edabdfef3ecb76f7d73a0c162", + "subject": "fix: unlink automation executions before cleaning expired reports", + "files": [ + "scheduler/dags/includes/constants.py", + "scheduler/tests/tasks/test_reports.py" + ], + "lines": 118 + }, + "benign_review": "accepted: scheduler cleanup of automation executions" + } + ], + "reviewed_by": "assistant review of each benign diff; selection of cases by the maintainer", + "frozen_on": "2026-09-24" +} diff --git a/benchmarks/independent/freeze.py b/benchmarks/independent/freeze.py new file mode 100644 index 0000000..10b50ec --- /dev/null +++ b/benchmarks/independent/freeze.py @@ -0,0 +1,171 @@ +"""Verify each reserved case by READING its source, and propose a benign change -- without scanning anything. + +Freezing the independent population (pre-push-safety-net 15.1) needs, per case: both pins present and in the +right order, the vulnerable sink confirmed where the advisory says it is, the fix touching that file, and a +benign change from the same history. This does exactly that with git and file reads. It never imports or runs +the analyzer, its queries, or any project tooling: a case this script touched is still unscanned. + +The benign change is a PROPOSAL for review, chosen mechanically: the first commit after the fix, on the first +-parent line, that changes source files of the case's language, does not touch the sink file, does not talk +about security in its message, and stays small. A reviewer confirms or replaces it before the status becomes +`frozen`. + +Usage: python benchmarks/independent/freeze.py [--cache DIR] [--only ID,...] +Writes: benchmarks/independent/freeze-v1.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import tomllib +from pathlib import Path + +HERE = Path(__file__).resolve().parent +POPULATION = HERE / "population-v1.toml" +OUT = HERE / "freeze-v1.json" +EXTENSIONS = { + "php": (".php",), + "javascript": (".js", ".mjs", ".cjs", ".jsx"), + "typescript": (".ts", ".tsx"), + "python": (".py",), +} +# A benign change must not be a security change in disguise. The first pass let through "Restrict access to +# routine logs to owners only", "querybuilder missing the valid_filter_params check" and "add access mode and +# scopes to API tokens" -- access control and input validation, which is exactly what a benign control must not be. +SECURITY_WORDS = re.compile( + r"secur|cve|ghsa|vuln|inject|xss|csrf|ssrf|sanitiz|escap|exploit|attack|auth|permission|cors|access|restrict|harden" + r"|token|scope|valid|filter|privileg|role|owner|password|secret|redirect|sql|query|login|log in|ldap|session|ssl|tls|verif|cert|crypt|hash|cookie", + re.I, +) +MAX_BENIGN_LINES = 200 + + +def git(repo: Path, *args: str, check: bool = True) -> str: + done = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, check=False) + if check and done.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {done.stderr.strip()[:300]}") + return done.stdout + + +def clone(url: str, target: Path) -> None: + if (target / ".git").is_dir(): + git(target, "fetch", "--quiet", "origin") + return + target.parent.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "clone", "--quiet", "--filter=blob:none", "--no-checkout", url, str(target)], check=True) + + +def blob(repo: Path, commit: str, path: str) -> bytes | None: + done = subprocess.run(["git", "-C", str(repo), "show", f"{commit}:{path}"], capture_output=True, check=False) + return done.stdout if done.returncode == 0 else None + + +def sink_parts(sink: str) -> tuple[str, list[str]]: + """The sink file and the function names the record claims, from `path :: A::b (...), c`.""" + path, _, rest = sink.partition("::") + names = re.findall(r"[A-Za-z_][A-Za-z0-9_]*(?=\s*(?:\(|,|$|;))", rest.split("(")[0] + ",") + functions = [n for n in names if n not in {"used", "by", "source", "callers", "and", "from", "the"}] + return path.strip(), functions + + +def benign(repo: Path, fixed: str, sink_file: str, extensions: tuple[str, ...]) -> dict[str, object] | None: + head = git(repo, "rev-parse", "origin/HEAD", check=False).strip() or git(repo, "rev-parse", "HEAD").strip() + commits = git(repo, "rev-list", "--first-parent", "--reverse", f"{fixed}..{head}", check=False).split()[:400] + for commit in commits: + message = git(repo, "log", "-1", "--format=%s%n%b", commit) + if SECURITY_WORDS.search(message): + continue + stat = git(repo, "show", "--numstat", "--format=", commit).splitlines() + files = [line.split("\t") for line in stat if line.count("\t") == 2] + if not files or any(f[2] == sink_file for f in files): + continue + source = [f for f in files if f[2].endswith(extensions)] + lines = sum(int(a) + int(b) for a, b, _ in files if a.isdigit() and b.isdigit()) + if source and 0 < lines <= MAX_BENIGN_LINES: + parent = git(repo, "rev-parse", f"{commit}^").strip() + return {"base": parent, "tip": commit, "subject": message.splitlines()[0][:120], "files": [f[2] for f in files], "lines": lines} + return None + + +def verify(case: dict[str, object], cache: Path) -> dict[str, object]: + url, vulnerable, fixed = str(case["repo"]), str(case["vulnerable"]), str(case["fixed"]) + repo = cache / str(case["id"]) + clone(url, repo) + record: dict[str, object] = {"id": case["id"], "repo": url, "vulnerable": vulnerable, "fixed": fixed, "problems": []} + problems: list[str] = record["problems"] # type: ignore[assignment] + for pin in (vulnerable, fixed): + if git(repo, "cat-file", "-t", pin, check=False).strip() != "commit": + problems.append(f"pin {pin[:12]} is not a commit in the repository") + if problems: + return record + ordered = subprocess.run(["git", "-C", str(repo), "merge-base", "--is-ancestor", vulnerable, fixed], check=False).returncode == 0 + record["vulnerable_is_ancestor_of_fixed"] = ordered + if not ordered: + problems.append("the vulnerable pin is not an ancestor of the fixed pin") + sink_file, functions = sink_parts(str(case["sink"])) + record["sink_file"], record["sink_functions"] = sink_file, functions + before, after = blob(repo, vulnerable, sink_file), blob(repo, fixed, sink_file) + if before is None: + problems.append(f"sink file {sink_file} is absent at the vulnerable pin") + else: + record["sink_sha256_vulnerable"] = hashlib.sha256(before).hexdigest() + text = before.decode(errors="replace") + evidence = str(case.get("sink_evidence", "")) + if evidence: + record["sink_evidence_found"] = evidence in text + if evidence not in text: + problems.append(f"sink evidence {evidence!r} is absent from {sink_file} at the vulnerable pin") + else: + missing = [f for f in functions if f.split("::")[-1] not in text] + record["sink_functions_missing"] = missing + if functions and len(missing) == len(functions): + problems.append(f"none of {functions} appears in {sink_file} at the vulnerable pin") + if after is not None: + record["sink_sha256_fixed"] = hashlib.sha256(after).hexdigest() + changed = git(repo, "diff", "--name-only", vulnerable, fixed).splitlines() + record["fix_changed_files"] = len(changed) + record["fix_touches_sink_file"] = sink_file in changed + fix_site = str(case.get("fix_site", "")).partition("::")[0].strip() + record["fix_touches_fix_site"] = bool(fix_site) and fix_site in changed + if sink_file not in changed and not record["fix_touches_fix_site"]: + problems.append(f"the fix changes neither {sink_file} nor a declared fix_site") + languages = [str(case["language"]), *[str(x) for x in case.get("also", [])]] # type: ignore[union-attr] + extensions = tuple(e for lang in languages for e in EXTENSIONS.get(lang, ())) + record["benign_proposal"] = benign(repo, fixed, sink_file, extensions) + if record["benign_proposal"] is None: + problems.append("no benign change found automatically; choose one by hand") + return record + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cache", type=Path, default=Path.home() / ".cache" / "openultrasast" / "independent") + parser.add_argument("--only", default="") + args = parser.parse_args() + cases = tomllib.loads(POPULATION.read_text())["case"] + only = {c.strip() for c in args.only.split(",") if c.strip()} + records = [] + for case in cases: + if only and case["id"] not in only: + continue + try: + record = verify(case, args.cache) + except (RuntimeError, subprocess.CalledProcessError) as error: + record = {"id": case["id"], "problems": [f"could not verify: {error}"]} + records.append(record) + print( + json.dumps( + {"id": record["id"], "problems": record["problems"], "benign": (record.get("benign_proposal") or {}).get("subject")} + ), + flush=True, + ) + OUT.write_text(json.dumps({"schema_version": 1, "population": POPULATION.name, "cases": records}, indent=2) + "\n") + return 0 if all(not r["problems"] for r in records) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/independent/population-v1.toml b/benchmarks/independent/population-v1.toml index 036ca30..98b20d8 100644 --- a/benchmarks/independent/population-v1.toml +++ b/benchmarks/independent/population-v1.toml @@ -18,7 +18,9 @@ # and the research notes are in the research log referenced below. schema_version = 1 -status = "reserved-unscanned" +status = "frozen" +frozen_on = "2026-09-24" +freeze_record = "freeze-v1.json" reserved_on = "2026-09-24" selected_by = "maintainer" research = "web research of GitHub advisories and fix diffs via the GitHub API; no clone, no scan" @@ -39,6 +41,7 @@ privilege = "unauthenticated (only with the custom usermeta table option on)" license = "GPL" notes = "Release pair 2.8.2 (e1885a73) -> 2.8.3 (a6a7d13a) spans 94 commits; the commit pair is the clean one. c61008bd is a larger follow-up refactor of the same file." +benign = { base = "13151ef558eb6998720cb1ee5f78b5f17535c16c", tip = "9130a3226ffec45569a2872ffac338200bd1bfcb", subject = "- wpcs;" } [[case]] id = "wp-learnpress-sqli" repo = "https://github.com/LearnPress/learnpress" @@ -51,8 +54,10 @@ fixed = "aaf42fe0c6bf05dcf2ff125750b7bd836e76fa3c" sink = "inc/Databases/class-lp-db.php :: LP_Database::execute (implode of only_fields into SELECT); source REST get_courses c_only_fields/c_fields" privilege = "unauthenticated (REST /wp-json/learnpress/v1/courses)" license = "GPL" -notes = "The fix also touches assets/src/js/loadAJAX.js: the change itself is multi-language." +fix_site = "inc/Models/Courses.php :: handle_params_for_query_courses (fields quoted as identifiers); the REST controller stops assigning c_only_fields directly" +notes = "The sink is where the SQL runs (class-lp-db.php:607 implode of only_fields); the fix is on the source side and does not change that file. It also touches assets/src/js/loadAJAX.js: the change itself is multi-language." +benign = { base = "aaf42fe0c6bf05dcf2ff125750b7bd836e76fa3c", tip = "179ed7760e6ae5fa46fbe9c5f1324278a42ece04", subject = "= 4.2.7.1 = ~ Added: lpOnElementReady function js" } [[case]] id = "phpmyfaq-captcha-ua-sqli" repo = "https://github.com/thorsten/phpMyFAQ" @@ -67,6 +72,7 @@ privilege = "unauthenticated (GET /api/captcha)" license = "MPL-2.0" notes = "The advisory's commit b9f25109 is the vulnerable master commit the reporter tested, not the fix. Release pair 4.1.1 (f791858b) -> 4.1.2 (66792f25)." +benign = { base = "bf781788ab6b5c2967a09d66216d7580778a2b5a", tip = "a4601ae5f01761dc0bb91b8c0c3901be91987ba3", subject = "build: updated version strings to 4.2" } [[case]] id = "yeswiki-bazar-form-sqli" repo = "https://github.com/YesWiki/yeswiki" @@ -81,6 +87,7 @@ privilege = "unauthenticated" license = "AGPL-3.0" notes = "Four-line fix: intval plus a Guard::isAllowed check. Default branch doryphore-dev." +benign = { base = "03f27fdde5f89f90de76cd3f2160cff82e7bffac", tip = "70d240c0b43f7e7b0bac55fc0431fb4c2d88b49c", subject = "fix(contact): send shared page by mail" } [[case]] id = "signalk-remote-conn-ssrf" repo = "https://github.com/SignalK/signalk-server" @@ -94,6 +101,7 @@ privilege = "unauthenticated by default (security not configured); admin otherwi license = "Apache-2.0" notes = "Fix adds a schema, an ssrfGuard module and requestId validation (~450 added lines, mostly guard code and tests)." +benign = { base = "8af993d45c898edfe28f820f1726b6fc33b27cad", tip = "07416b79601c5d901f671f70a94bd57bed90bad8", subject = "fix: allow multiple deltaHandler next calls (#2818)" } [[case]] id = "saltcorn-sync-sqli" repo = "https://github.com/saltcorn/saltcorn" @@ -108,6 +116,7 @@ privilege = "authenticated low-privilege user" license = "MIT" notes = "Large multi-package repository; the fix is on the main line (v1.4.6 is a separate backport), so use the commit pair." +benign = { base = "5c619403c40447bf56ce662570d3a3ffbf781019", tip = "df114765fd36d280395b2129f1703fa98c712d06", subject = "updated at in trigger.findAllWithTableName" } [[case]] id = "mongo-express-bson-rce" repo = "https://github.com/mongo-express/mongo-express" @@ -121,6 +130,7 @@ privilege = "authenticated (HTTP basic auth, default admin:pass)" license = "MIT" notes = "Code injection, not SQL. The advisory's first reference 7d365141 is only the v0.54.0 release commit." +benign = { base = "9550ae2529258b47a2b637d8e64e69a0d4d38ad8", tip = "5b9b34f8553d004ceb7b666bbf566b406a2a3153", subject = "Merge pull request #538 from damien-malescot/master" } [[case]] id = "mockoon-admin-api-cors" repo = "https://github.com/mockoon/mockoon" @@ -130,10 +140,12 @@ advisory = ["CVE-2026-59148", "GHSA-rqx4-3f6q-3x2v"] vulnerable = "b42bdfb7f82e83f0e81bea8e6fe41adf5ec82585" fixed = "c420b5a56918475b8663977b51e5f986e45b3299" sink = "packages/commons-server/src/libs/server/admin-api.ts :: admin-API middleware (Access-Control-Allow-Origin: *)" +sink_evidence = "'Access-Control-Allow-Origin': '*'" privilege = "unauthenticated" license = "MIT" notes = "Weaker fit: the fix commit touches 34 files; only one hunk in admin-api.ts is the CORS fix." +benign = { base = "de69f5e30f89e17918855d147f4e8c055d5b2ba9", tip = "895961070b55a14d290ad50232ab9c6d800c3cad", subject = "Fix serverless lib returning a 404 when no body is present (#2274)" } [[case]] id = "alerta-query-sqli" repo = "https://github.com/alerta/alerta" @@ -147,6 +159,7 @@ privilege = "API user; possibly unauthenticated if AUTH_REQUIRED defaults to Fal license = "Apache-2.0" notes = "The advisory's aeba85a3 introduced the parser; the fix is fdd52cd1." +benign = { base = "b7111d23ae9aefb5064ef107976f62e3836217b6", tip = "71c098d8bb8091e76edaefeef4662eb77c753ce7", subject = "fix(deps): replace pkg_resources with importlib.metadata (#2041)" } [[case]] id = "wger-next-open-redirect" repo = "https://github.com/wger-project/wger" @@ -160,6 +173,7 @@ privilege = "use_browser_language: unauthenticated; trainer_login: authenticated license = "AGPL-3.0" notes = "No CVE. A later commit cfa27a7f fixes the same pattern in core/middleware.py; not part of this fix." +benign = { base = "47d646c24178783eb6787ecf26e79dc8f4a394c6", tip = "c1144533bf9bca487ff905d925b567151d89c1aa", subject = "Formatting" } [[case]] id = "opencve-notification-ssrf" repo = "https://github.com/opencve/opencve" @@ -172,3 +186,4 @@ sink = "web/projects/notifications.py :: _http_post_json (webhook and Slack test privilege = "authenticated (can configure project notifications)" license = "unverified (GitHub reports NOASSERTION; possibly non-OSS BSL) -- verify before publishing any result" notes = "Selected by the maintainer despite the licence question; results must not be redistributed until it is checked." +benign = { base = "0c8101082ba20628b28fcec9c4611745d32202c2", tip = "965dc348e225308edabdfef3ecb76f7d73a0c162", subject = "fix: unlink automation executions before cleaning expired reports" } diff --git a/tests/test_independent_population.py b/tests/test_independent_population.py index c295dcb..59bd2d7 100644 --- a/tests/test_independent_population.py +++ b/tests/test_independent_population.py @@ -34,6 +34,13 @@ def test_the_reservation_is_well_formed() -> None: assert case["vulnerable"] != case["fixed"], case["id"] assert case["repo"].startswith("https://github.com/"), case["id"] assert case["advisory"] and case["sink"] and case["privilege"], case["id"] + if data["status"] == "frozen": + # Frozen means every case carries its benign control: a change from the same history, reviewed as not a + # security change, whose two sides a correct analyzer must not tell apart. + for case in cases: + benign = case.get("benign") or {} + assert SHA.fullmatch(benign.get("base", "")) and SHA.fullmatch(benign.get("tip", "")), case["id"] + assert (POPULATION.parent / data["freeze_record"]).is_file() languages = {case["language"] for case in cases} assert {"php", "python"} <= languages and languages & {"javascript", "typescript"}