From faf4f643df8900e4dee1a60913e28f6efdec5897 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:57:02 +0200 Subject: [PATCH] measure(sqli): adjudicate a 1-in-5 sample of PMPro SQL injection findings 48 findings, every 5th reviewed against source: 2 true (both privileged), 8 false; 6 of the 8 share one cause, object fields treated as attacker input when they are loaded from the database. benchmarks/push/finding_dump.py writes every finding of chosen families for review. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- .../2026-09-23-pmpro-sqli-adjudication.json | 25 +++++ benchmarks/push/finding_dump.py | 95 +++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 benchmarks/measurements/2026-09-23-pmpro-sqli-adjudication.json create mode 100644 benchmarks/push/finding_dump.py diff --git a/benchmarks/measurements/2026-09-23-pmpro-sqli-adjudication.json b/benchmarks/measurements/2026-09-23-pmpro-sqli-adjudication.json new file mode 100644 index 0000000..d5ce485 --- /dev/null +++ b/benchmarks/measurements/2026-09-23-pmpro-sqli-adjudication.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "experiment": "pmpro-sqli-precision-sample-v1", + "date": "2026-09-23", + "subject": "Paid Memberships Pro 2.9.7 (pin 1a900be3f09f), development subject", + "scan": {"source_commit": "6b85a60", "families": ["injection"], "regions": 4461, "questions": 382, "completed": 272, "findings": 48, "all_rungs": "model_entailed", "seconds": 2935.9, "tool": "benchmarks/push/finding_dump.py"}, + "sampling": "every 5th finding in site order (indices 0,5,...,45), requested by the maintainer", + "standard": "TP = attacker-controlled request data reaches SQL text without prepare, integer cast, or esc_sql inside quotes (sanitize_text_field does not count). TP-privileged = same, reachable only by an administrator/privileged capability. FP-guarded = escaped/cast/allow-listed/bound first. FP-not-attacker = value from the database, a constant or internal state.", + "sample": [ + {"i": 0, "site": "adminpages/reports/memberships.php:313", "verdict": "FP-guarded", "reason": "every interpolated value is intval/esc_sql-in-quotes/constant; witness source $type never enters SQL except as an extra apply_filters argument", "fp_class": "apply_filters_extra_arg"}, + {"i": 5, "site": "classes/class-pmpro-discount-codes.php:154", "verdict": "TP-privileged", "reason": "REST POST /pmpro/v1/discount_code: code = sanitize_text_field(param), interpolated in quotes in save(); needs pmpro_discountcodes capability. Witness names $this->expires, not $this->code"}, + {"i": 10, "site": "classes/class-pmpro-members-list-table.php:384", "verdict": "FP-guarded", "reason": "orderby via allow-list map sanitize_orderby, order via ==, search_key in_array allow-list, $s esc_sql in quotes", "fp_class": "allowlist_lookup"}, + {"i": 15, "site": "classes/class.memberorder.php:1122", "verdict": "FP-not-attacker", "reason": "$this->membership_id loaded from the order's DB row; witness source $update_order->BillingFrequency is internal state (upstream hardened it with prepare in 2.9.8)", "fp_class": "object_field_source"}, + {"i": 20, "site": "classes/class.memberorder.php:1633", "verdict": "FP-not-attacker", "reason": "$this->user_id / membership_id internal order state", "fp_class": "object_field_source"}, + {"i": 25, "site": "classes/class.memberorder.php:705", "verdict": "FP-not-attacker", "reason": "$this->user_id = $dbobj->user_id from the DB row; the injectable query is the separate id lookup at line 671", "fp_class": "object_field_source"}, + {"i": 30, "site": "classes/class.pmproemail.php:262", "verdict": "FP-not-attacker", "reason": "$user->ID and old level ids are internal", "fp_class": "object_field_source"}, + {"i": 35, "site": "classes/gateways/class.pmprogateway_twocheckout.php:240", "verdict": "FP-not-attacker", "reason": "(int) cast code_id, WordPress user id, internal order id; witness source $morder->user_id", "fp_class": "object_field_source"}, + {"i": 40, "site": "includes/profile.php:344", "verdict": "TP-privileged", "reason": "$user_ID = $_REQUEST['user_id'] raw, interpolated in quotes; needs the membership-level capability. Sibling finding at line 337 (raw expires_hour/minute) is also real"}, + {"i": 45, "site": "includes/profile.php:515", "verdict": "FP-not-attacker", "reason": "$invoice->code_id from a DB row", "fp_class": "object_field_source"} + ], + "tally": {"TP": 0, "TP_privileged": 2, "FP_guarded": 2, "FP_not_attacker": 6, "precision": "2/10", "precision_95ci_wilson": "0.06-0.51"}, + "false_positive_classes": {"object_field_source": 6, "apply_filters_extra_arg": 1, "allowlist_lookup": 1}, + "reading": "SQL injection precision on this sample is 20% against a 95% gate. Six of eight false positives share one cause: an object field ($this->x, $obj->x) is treated as attacker input although, in these PHP classes, it is loaded from the database or set internally. The real findings are both privileged (admin capability); none in the sample is reachable unauthenticated. The declared target (class.memberorder.php:936) is a true positive outside the sample.", + "limits": ["One subject, development data; a 10-finding sample has a wide interval.", "Privilege and reachability judged from capability checks in code, not by exploitation."] +} diff --git a/benchmarks/push/finding_dump.py b/benchmarks/push/finding_dump.py new file mode 100644 index 0000000..f7da829 --- /dev/null +++ b/benchmarks/push/finding_dump.py @@ -0,0 +1,95 @@ +"""Every finding of chosen families on one checkout, whole, for a human to adjudicate. + +The family census keeps ten examples per family, which is enough to say a family establishes something and far +too few to say whether what it establishes is TRUE. Precision is the release gate (95%) and had never been +measured on PHP: 224 findings on Paid Memberships Pro, none reviewed. This runs the production discovery and scan +restricted to the families asked for, and writes every finding with its site, rung and witness, sorted by site so +a systematic sample (every n-th) is reproducible. + +Usage: python finding_dump.py --root --families injection --out findings.json [--deadline 3600] +""" + +from __future__ import annotations + +import argparse +import dataclasses +import json +import time +from pathlib import Path + +from openultrasast.cpg.backend import JoernBackend +from openultrasast.mapping import analyze_entry_points +from openultrasast.model.contracts import ExecutionBudget +from openultrasast.model.regions import regions_for +from openultrasast.model.scan import ScanBudget, scan_repository +from openultrasast.model.shipped import declared_sources +from openultrasast.preprocess import build_file_target, enumerate_source_files + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--families", default="injection") + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--deadline", type=float, default=3600.0) + parser.add_argument("--max-regions", type=int, default=5000) + args = parser.parse_args() + wanted = tuple(f.strip() for f in args.families.split(",") if f.strip()) + + files = enumerate_source_files(args.root) + opened = sum(p.stat().st_size for p in files if p.is_file()) + if not files or not opened: + raise SystemExit(f"read nothing under {args.root}") + targets = [build_file_target(args.root, p) for p in files] + regions = [ + dataclasses.replace(r, families=tuple(f for f in r.families if f in wanted)) + for r in regions_for(analyze_entry_points(args.root, targets), targets, shipped=declared_sources(args.root)) + ] + regions = [r for r in regions if r.families] + print(f"read {len(files)} files, {opened} bytes; {len(regions)} regions offer {wanted}", flush=True) + + started = time.monotonic() + budget = ExecutionBudget(time.monotonic() + args.deadline, 5.0) + result = scan_repository( + args.root, + regions[: args.max_regions], + backend=JoernBackend(execution_budget=budget), + budget=ScanBudget(max_model_calls=0, max_regions=args.max_regions, order_by_evidence=True), + execution_budget=budget, + ranking_mode="evidence", + unit="repository", + population_complete=True, + ) + outcomes = result.question_outcomes + findings = sorted( + ( + { + "site": f.site, + "family": f.family, + "rung": str(getattr(f.rung, "value", f.rung)), + "witness": f.witness, + "reached_from": f.reached_from, + } + for f in result.findings + if f.family in wanted + ), + key=lambda f: f["site"], + ) + record = { + "root": str(args.root), + "families": wanted, + "seconds": round(time.monotonic() - started, 1), + "questions": len(outcomes), + "completed": sum(1 for o in outcomes if o.status == "completed"), + "degradations": sorted({str(d.get("reason")) for d in result.degradations}), + "findings": findings, + } + args.out.write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps({k: v for k, v in record.items() if k != "findings"} | {"findings": len(findings)}), flush=True) + if not outcomes: + raise SystemExit("no question was asked: the dump measured nothing") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())