mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
measure(sqli): adjudicate a 1-in-5 sample of PMPro SQL injection findings
48 findings, every 5th reviewed against source: 2 true (both privileged), 8 false; 6 of the 8 share one cause, object fields treated as attacker input when they are loaded from the database. benchmarks/push/finding_dump.py writes every finding of chosen families for review. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
c82ce7fa44
commit
faf4f643df
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"experiment": "pmpro-sqli-precision-sample-v1",
|
||||
"date": "2026-09-23",
|
||||
"subject": "Paid Memberships Pro 2.9.7 (pin 1a900be3f09f), development subject",
|
||||
"scan": {"source_commit": "6b85a60", "families": ["injection"], "regions": 4461, "questions": 382, "completed": 272, "findings": 48, "all_rungs": "model_entailed", "seconds": 2935.9, "tool": "benchmarks/push/finding_dump.py"},
|
||||
"sampling": "every 5th finding in site order (indices 0,5,...,45), requested by the maintainer",
|
||||
"standard": "TP = attacker-controlled request data reaches SQL text without prepare, integer cast, or esc_sql inside quotes (sanitize_text_field does not count). TP-privileged = same, reachable only by an administrator/privileged capability. FP-guarded = escaped/cast/allow-listed/bound first. FP-not-attacker = value from the database, a constant or internal state.",
|
||||
"sample": [
|
||||
{"i": 0, "site": "adminpages/reports/memberships.php:313", "verdict": "FP-guarded", "reason": "every interpolated value is intval/esc_sql-in-quotes/constant; witness source $type never enters SQL except as an extra apply_filters argument", "fp_class": "apply_filters_extra_arg"},
|
||||
{"i": 5, "site": "classes/class-pmpro-discount-codes.php:154", "verdict": "TP-privileged", "reason": "REST POST /pmpro/v1/discount_code: code = sanitize_text_field(param), interpolated in quotes in save(); needs pmpro_discountcodes capability. Witness names $this->expires, not $this->code"},
|
||||
{"i": 10, "site": "classes/class-pmpro-members-list-table.php:384", "verdict": "FP-guarded", "reason": "orderby via allow-list map sanitize_orderby, order via ==, search_key in_array allow-list, $s esc_sql in quotes", "fp_class": "allowlist_lookup"},
|
||||
{"i": 15, "site": "classes/class.memberorder.php:1122", "verdict": "FP-not-attacker", "reason": "$this->membership_id loaded from the order's DB row; witness source $update_order->BillingFrequency is internal state (upstream hardened it with prepare in 2.9.8)", "fp_class": "object_field_source"},
|
||||
{"i": 20, "site": "classes/class.memberorder.php:1633", "verdict": "FP-not-attacker", "reason": "$this->user_id / membership_id internal order state", "fp_class": "object_field_source"},
|
||||
{"i": 25, "site": "classes/class.memberorder.php:705", "verdict": "FP-not-attacker", "reason": "$this->user_id = $dbobj->user_id from the DB row; the injectable query is the separate id lookup at line 671", "fp_class": "object_field_source"},
|
||||
{"i": 30, "site": "classes/class.pmproemail.php:262", "verdict": "FP-not-attacker", "reason": "$user->ID and old level ids are internal", "fp_class": "object_field_source"},
|
||||
{"i": 35, "site": "classes/gateways/class.pmprogateway_twocheckout.php:240", "verdict": "FP-not-attacker", "reason": "(int) cast code_id, WordPress user id, internal order id; witness source $morder->user_id", "fp_class": "object_field_source"},
|
||||
{"i": 40, "site": "includes/profile.php:344", "verdict": "TP-privileged", "reason": "$user_ID = $_REQUEST['user_id'] raw, interpolated in quotes; needs the membership-level capability. Sibling finding at line 337 (raw expires_hour/minute) is also real"},
|
||||
{"i": 45, "site": "includes/profile.php:515", "verdict": "FP-not-attacker", "reason": "$invoice->code_id from a DB row", "fp_class": "object_field_source"}
|
||||
],
|
||||
"tally": {"TP": 0, "TP_privileged": 2, "FP_guarded": 2, "FP_not_attacker": 6, "precision": "2/10", "precision_95ci_wilson": "0.06-0.51"},
|
||||
"false_positive_classes": {"object_field_source": 6, "apply_filters_extra_arg": 1, "allowlist_lookup": 1},
|
||||
"reading": "SQL injection precision on this sample is 20% against a 95% gate. Six of eight false positives share one cause: an object field ($this->x, $obj->x) is treated as attacker input although, in these PHP classes, it is loaded from the database or set internally. The real findings are both privileged (admin capability); none in the sample is reachable unauthenticated. The declared target (class.memberorder.php:936) is a true positive outside the sample.",
|
||||
"limits": ["One subject, development data; a 10-finding sample has a wide interval.", "Privilege and reachability judged from capability checks in code, not by exploitation."]
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Every finding of chosen families on one checkout, whole, for a human to adjudicate.
|
||||
|
||||
The family census keeps ten examples per family, which is enough to say a family establishes something and far
|
||||
too few to say whether what it establishes is TRUE. Precision is the release gate (95%) and had never been
|
||||
measured on PHP: 224 findings on Paid Memberships Pro, none reviewed. This runs the production discovery and scan
|
||||
restricted to the families asked for, and writes every finding with its site, rung and witness, sorted by site so
|
||||
a systematic sample (every n-th) is reproducible.
|
||||
|
||||
Usage: python finding_dump.py --root <checkout> --families injection --out findings.json [--deadline 3600]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.cpg.backend import JoernBackend
|
||||
from openultrasast.mapping import analyze_entry_points
|
||||
from openultrasast.model.contracts import ExecutionBudget
|
||||
from openultrasast.model.regions import regions_for
|
||||
from openultrasast.model.scan import ScanBudget, scan_repository
|
||||
from openultrasast.model.shipped import declared_sources
|
||||
from openultrasast.preprocess import build_file_target, enumerate_source_files
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, required=True)
|
||||
parser.add_argument("--families", default="injection")
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--deadline", type=float, default=3600.0)
|
||||
parser.add_argument("--max-regions", type=int, default=5000)
|
||||
args = parser.parse_args()
|
||||
wanted = tuple(f.strip() for f in args.families.split(",") if f.strip())
|
||||
|
||||
files = enumerate_source_files(args.root)
|
||||
opened = sum(p.stat().st_size for p in files if p.is_file())
|
||||
if not files or not opened:
|
||||
raise SystemExit(f"read nothing under {args.root}")
|
||||
targets = [build_file_target(args.root, p) for p in files]
|
||||
regions = [
|
||||
dataclasses.replace(r, families=tuple(f for f in r.families if f in wanted))
|
||||
for r in regions_for(analyze_entry_points(args.root, targets), targets, shipped=declared_sources(args.root))
|
||||
]
|
||||
regions = [r for r in regions if r.families]
|
||||
print(f"read {len(files)} files, {opened} bytes; {len(regions)} regions offer {wanted}", flush=True)
|
||||
|
||||
started = time.monotonic()
|
||||
budget = ExecutionBudget(time.monotonic() + args.deadline, 5.0)
|
||||
result = scan_repository(
|
||||
args.root,
|
||||
regions[: args.max_regions],
|
||||
backend=JoernBackend(execution_budget=budget),
|
||||
budget=ScanBudget(max_model_calls=0, max_regions=args.max_regions, order_by_evidence=True),
|
||||
execution_budget=budget,
|
||||
ranking_mode="evidence",
|
||||
unit="repository",
|
||||
population_complete=True,
|
||||
)
|
||||
outcomes = result.question_outcomes
|
||||
findings = sorted(
|
||||
(
|
||||
{
|
||||
"site": f.site,
|
||||
"family": f.family,
|
||||
"rung": str(getattr(f.rung, "value", f.rung)),
|
||||
"witness": f.witness,
|
||||
"reached_from": f.reached_from,
|
||||
}
|
||||
for f in result.findings
|
||||
if f.family in wanted
|
||||
),
|
||||
key=lambda f: f["site"],
|
||||
)
|
||||
record = {
|
||||
"root": str(args.root),
|
||||
"families": wanted,
|
||||
"seconds": round(time.monotonic() - started, 1),
|
||||
"questions": len(outcomes),
|
||||
"completed": sum(1 for o in outcomes if o.status == "completed"),
|
||||
"degradations": sorted({str(d.get("reason")) for d in result.degradations}),
|
||||
"findings": findings,
|
||||
}
|
||||
args.out.write_text(json.dumps(record, indent=2) + "\n")
|
||||
print(json.dumps({k: v for k, v in record.items() if k != "findings"} | {"findings": len(findings)}), flush=True)
|
||||
if not outcomes:
|
||||
raise SystemExit("no question was asked: the dump measured nothing")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user