mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(ruleset): PHP quick-mode rules from the engine's vocabulary, measured on the development corpus
Quick mode had no PHP rules, so a PHP quick scan read the files and reported a plausible zero. ruleset/php/rules.toml adds 13 line/statement rules built on semantic/php.toml's sinks, sources and sanitizers (sql/wpdb, command, code_eval, unserialize, echo, file_inclusion, filesystem, outbound_request, redirect). 9 are enabled and 4 are shadow (sql-call-composition, code-eval, unserialize, xss-echo-input). A rule is shadow when it fired on the corpus but never at a known site. precision_estimate is the measured lower bound. Measured in-sample on the development corpus only (pmpro introduce/repair/benign, population-v1 PHP cases; population-v2 was not read). Known site hit on the vulnerable pin in 4 of 5 cases (LearnPress missed). The hit disappears on the fixed pin for pmpro and YesWiki. Numbers, per-pin files/bytes read and the engine comparison are in benchmarks/measurements/2026-09-30-php-quick-rules/. - fixtures php-vulnerable / php-benign plus manifests: ousast benchmark covers PHP (9/14 reported, 13/14 including shadow; the benign twin records 2 known false alerts) - PHP `#` comments are comments; `#[...]` attributes are not - the alerts task now reports PHP as quick coverage; alerts-engine keeps running the engine for PHP next to the quick rules (ENGINE_ALONGSIDE_QUICK) - other languages' benchmark findings and metrics are unchanged Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
45d02ecf20
commit
7927d6e539
@@ -0,0 +1,89 @@
|
||||
<?php
|
||||
// Benign twin of benchmarks/fixtures/php-vulnerable/app.php: the same handlers, each one repaired the way its
|
||||
// upstream fix did it. Every quick-mode finding here is a false alert (benchmarks/manifests/php-benign.toml).
|
||||
|
||||
$conn = mysqli_connect('localhost', 'app', 'app', 'app');
|
||||
|
||||
function order_by_code($code)
|
||||
{
|
||||
global $wpdb;
|
||||
return $wpdb->get_var($wpdb->prepare("SELECT id FROM {$wpdb->prefix}orders WHERE code = %s LIMIT 1", $code));
|
||||
}
|
||||
|
||||
function find_user($conn)
|
||||
{
|
||||
$stmt = mysqli_prepare($conn, 'SELECT * FROM users WHERE name = ?');
|
||||
mysqli_stmt_bind_param($stmt, 's', $_GET['name']);
|
||||
mysqli_stmt_execute($stmt);
|
||||
return mysqli_stmt_get_result($stmt);
|
||||
}
|
||||
|
||||
function members_sorted($order)
|
||||
{
|
||||
$order = in_array(strtoupper($order), array('ASC', 'DESC'), true) ? strtoupper($order) : 'ASC';
|
||||
return ' ORDER BY u.user_login ' . ('DESC' === $order ? 'DESC' : 'ASC') . ' ';
|
||||
}
|
||||
|
||||
function create_form($data)
|
||||
{
|
||||
return 'INSERT INTO forms (id, label) VALUES (' . intval($data['id']) . ', "x")';
|
||||
}
|
||||
|
||||
function captcha_cleanup($db, $userAgent)
|
||||
{
|
||||
$delete = sprintf(
|
||||
"DELETE FROM captcha WHERE useragent = '%s'",
|
||||
$db->escape($userAgent)
|
||||
);
|
||||
$db->query($delete);
|
||||
}
|
||||
|
||||
function ping()
|
||||
{
|
||||
system('ping -c 1 ' . escapeshellarg($_GET['host']));
|
||||
}
|
||||
|
||||
function calculate($formula)
|
||||
{
|
||||
return is_numeric($formula) ? (float) $formula : 0.0;
|
||||
}
|
||||
|
||||
function restore_session()
|
||||
{
|
||||
return json_decode($_COOKIE['session'], true);
|
||||
}
|
||||
|
||||
function greet()
|
||||
{
|
||||
echo 'Hello ' . htmlspecialchars($_GET['name'], ENT_QUOTES);
|
||||
}
|
||||
|
||||
function page()
|
||||
{
|
||||
$pages = array('home', 'about');
|
||||
include in_array($_GET['page'], $pages, true) ? $_GET['page'] . '.php' : 'home.php';
|
||||
}
|
||||
|
||||
function download()
|
||||
{
|
||||
readfile('/var/www/files/' . basename($_GET['file']));
|
||||
}
|
||||
|
||||
function fetch_preview()
|
||||
{
|
||||
$url = filter_var($_GET['url'], FILTER_VALIDATE_URL);
|
||||
$ch = curl_init(wp_http_validate_url($url) ? $url : 'about:blank');
|
||||
return curl_exec($ch);
|
||||
}
|
||||
|
||||
function go_back()
|
||||
{
|
||||
wp_safe_redirect(wp_validate_redirect(wp_unslash($_GET['next']), home_url()));
|
||||
}
|
||||
|
||||
function find_order($conn)
|
||||
{
|
||||
$id = intval($_POST['id']);
|
||||
$sql = 'SELECT * FROM orders WHERE id = ' . $id;
|
||||
return mysqli_query($conn, $sql);
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
<?php
|
||||
// Intentionally vulnerable PHP fixture for the quick-mode PHP rules. Do not deploy.
|
||||
// Each handler has one labeled weakness (benchmarks/manifests/php-vulnerable.toml); the fixed twin is
|
||||
// benchmarks/fixtures/php-benign/app.php.
|
||||
|
||||
$conn = mysqli_connect('localhost', 'app', 'app', 'app');
|
||||
|
||||
function order_by_code($code)
|
||||
{
|
||||
global $wpdb;
|
||||
return $wpdb->get_var("SELECT id FROM {$wpdb->prefix}orders WHERE code = '" . $code . "' LIMIT 1");
|
||||
}
|
||||
|
||||
function find_user($conn)
|
||||
{
|
||||
return mysqli_query($conn, "SELECT * FROM users WHERE name = '" . $_GET['name'] . "'");
|
||||
}
|
||||
|
||||
function members_sorted($order)
|
||||
{
|
||||
$order = esc_sql($order); // escaping does not protect an unquoted ORDER BY
|
||||
return " ORDER BY u.user_login {$order} ";
|
||||
}
|
||||
|
||||
function create_form($data)
|
||||
{
|
||||
return 'INSERT INTO forms (id, label) VALUES (' . $data['id'] . ', "x")';
|
||||
}
|
||||
|
||||
function captcha_cleanup($db, $userAgent)
|
||||
{
|
||||
$delete = sprintf(
|
||||
"DELETE FROM captcha WHERE useragent = '%s'",
|
||||
$userAgent
|
||||
);
|
||||
$db->query($delete);
|
||||
}
|
||||
|
||||
function ping()
|
||||
{
|
||||
system('ping -c 1 ' . $_GET['host']);
|
||||
}
|
||||
|
||||
function calculate($formula)
|
||||
{
|
||||
eval('$value = ' . $formula . ';');
|
||||
return $value;
|
||||
}
|
||||
|
||||
function restore_session()
|
||||
{
|
||||
return unserialize($_COOKIE['session']);
|
||||
}
|
||||
|
||||
function greet()
|
||||
{
|
||||
echo 'Hello ' . $_GET['name'];
|
||||
}
|
||||
|
||||
function page()
|
||||
{
|
||||
include $_GET['page'] . '.php';
|
||||
}
|
||||
|
||||
function download()
|
||||
{
|
||||
readfile('/var/www/files/' . $_GET['file']);
|
||||
}
|
||||
|
||||
function fetch_preview()
|
||||
{
|
||||
$ch = curl_init($_GET['url']);
|
||||
return curl_exec($ch);
|
||||
}
|
||||
|
||||
function go_back()
|
||||
{
|
||||
header('Location: ' . $_GET['next']);
|
||||
}
|
||||
|
||||
// Not visible to a line rule: the value reaches the sink through a variable on another line.
|
||||
function find_order($conn)
|
||||
{
|
||||
$id = $_POST['id'];
|
||||
$sql = "SELECT * FROM orders WHERE id = '" . $id . "'";
|
||||
return mysqli_query($conn, $sql);
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
<?php
|
||||
// Safe equivalents. None of these lines should produce a finding.
|
||||
|
||||
function order_by_code_safe($code)
|
||||
{
|
||||
global $wpdb;
|
||||
return $wpdb->get_var($wpdb->prepare("SELECT id FROM {$wpdb->prefix}orders WHERE code = %s LIMIT 1", $code));
|
||||
}
|
||||
|
||||
function orders_in_safe($ids)
|
||||
{
|
||||
global $wpdb;
|
||||
$ids_format = implode(', ', array_fill(0, count($ids), '%d'));
|
||||
return $wpdb->prepare('SELECT * FROM orders WHERE id IN (' . $ids_format . ')', $ids);
|
||||
}
|
||||
|
||||
function page_limit_safe($page)
|
||||
{
|
||||
return ' LIMIT ' . intval($page) . ', 20';
|
||||
}
|
||||
|
||||
function quoted_escaped_safe($name)
|
||||
{
|
||||
return "SELECT * FROM users WHERE name = '" . esc_sql($name) . "'";
|
||||
}
|
||||
|
||||
function ping_safe()
|
||||
{
|
||||
system('ping -c 1 ' . escapeshellarg($_GET['host']));
|
||||
}
|
||||
|
||||
function version_safe()
|
||||
{
|
||||
exec('git describe --tags', $output);
|
||||
return $output;
|
||||
}
|
||||
|
||||
function greet_safe()
|
||||
{
|
||||
echo 'Hello ' . htmlspecialchars($_GET['name'], ENT_QUOTES);
|
||||
}
|
||||
|
||||
function download_safe()
|
||||
{
|
||||
readfile('/var/www/files/' . basename($_GET['file']));
|
||||
}
|
||||
|
||||
function restore_safe($raw)
|
||||
{
|
||||
return unserialize($raw, ['allowed_classes' => false]);
|
||||
}
|
||||
|
||||
function query_args_safe()
|
||||
{
|
||||
$args = [];
|
||||
$args['include'] = [(int) $_REQUEST['p']];
|
||||
return $args;
|
||||
}
|
||||
|
||||
function comment_only()
|
||||
{
|
||||
// system('rm -rf ' . $_GET['dir']) would be dangerous but this is a comment
|
||||
# eval($_POST['code']) is a comment too
|
||||
return 'ok';
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
name = "php-benign"
|
||||
# The repaired twin of php-vulnerable: no expected findings, so every quick-mode finding is a false alert.
|
||||
language = "php_web"
|
||||
frameworks = ["wordpress"]
|
||||
setup = []
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/php-benign"
|
||||
@@ -0,0 +1,136 @@
|
||||
name = "php-vulnerable"
|
||||
language = "php_web"
|
||||
frameworks = ["wordpress"]
|
||||
setup = []
|
||||
modes = ["quick", "standard"]
|
||||
known_noise = []
|
||||
baselines = []
|
||||
|
||||
[source]
|
||||
type = "local"
|
||||
path = "../fixtures/php-vulnerable"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "app.php"
|
||||
line = 11
|
||||
rule_id = "php-wpdb-sql-composition"
|
||||
sink = "get_var"
|
||||
evidence = "wpdb get_var query concatenates the order code without prepare"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "app.php"
|
||||
line = 16
|
||||
rule_id = "php-sql-call-composition"
|
||||
sink = "mysqli_query"
|
||||
evidence = "mysqli_query composes the query from request input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "app.php"
|
||||
line = 22
|
||||
rule_id = "php-sql-unquoted-interpolation"
|
||||
sink = "ORDER"
|
||||
evidence = "esc_sql value interpolated into an unquoted ORDER BY"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "app.php"
|
||||
line = 27
|
||||
rule_id = "php-sql-unquoted-concat"
|
||||
sink = "VALUES"
|
||||
evidence = "form id concatenated unquoted into VALUES"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "app.php"
|
||||
line = 32
|
||||
rule_id = "php-sql-sprintf-unescaped"
|
||||
sink = "sprintf"
|
||||
evidence = "sprintf SQL with an unescaped quoted %s user agent"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "app.php"
|
||||
line = 41
|
||||
rule_id = "php-command-injection"
|
||||
sink = "system"
|
||||
evidence = "system runs a command built from request input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-95"
|
||||
class = "code injection"
|
||||
path = "app.php"
|
||||
line = 46
|
||||
rule_id = "php-code-eval"
|
||||
sink = "eval"
|
||||
evidence = "eval executes a formula string"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-502"
|
||||
class = "insecure deserialization"
|
||||
path = "app.php"
|
||||
line = 52
|
||||
rule_id = "php-unserialize"
|
||||
sink = "unserialize"
|
||||
evidence = "unserialize of a cookie"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-79"
|
||||
class = "cross-site scripting"
|
||||
path = "app.php"
|
||||
line = 57
|
||||
rule_id = "php-xss-echo-input"
|
||||
sink = "echo"
|
||||
evidence = "request input echoed without escaping"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-98"
|
||||
class = "file inclusion"
|
||||
path = "app.php"
|
||||
line = 62
|
||||
rule_id = "php-file-inclusion-input"
|
||||
sink = "include"
|
||||
evidence = "include of a request-chosen path"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-22"
|
||||
class = "path traversal"
|
||||
path = "app.php"
|
||||
line = 67
|
||||
rule_id = "php-path-input"
|
||||
sink = "readfile"
|
||||
evidence = "readfile of a request-chosen path"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-918"
|
||||
class = "server-side request forgery"
|
||||
path = "app.php"
|
||||
line = 72
|
||||
rule_id = "php-ssrf-input"
|
||||
sink = "curl_init"
|
||||
evidence = "curl request to a request-chosen URL"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-601"
|
||||
class = "open redirect"
|
||||
path = "app.php"
|
||||
line = 78
|
||||
rule_id = "php-open-redirect"
|
||||
sink = "header"
|
||||
evidence = "Location header from request input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-89"
|
||||
class = "sql injection"
|
||||
path = "app.php"
|
||||
line = 86
|
||||
sink = "mysqli_query"
|
||||
evidence = "query built from a request value on an earlier line (not visible to a line rule)"
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Measure the PHP quick-mode rules on the PHP development corpus (never population-v2).
|
||||
|
||||
Pins: pmpro vulnerable / fixed / benign (benchmarks/push/inputs.json, role development) and the spent
|
||||
population-v1 PHP cases' vulnerable and fixed pins, exported beforehand with benchmarks/independent/evaluate.py's
|
||||
`export` into <corpus>/checkouts/<case>/<label> (pmpro from the repo cache, benign = vulnerable + the declared
|
||||
authored edit).
|
||||
|
||||
Known sites: pmpro's is the declared CVE-2023-23488 function (getMemberOrderByCode) on each pin; a v1 case's are
|
||||
its fix hunks (vulnerable side on the vulnerable pin, fixed side on the fixed pin; test paths dropped) widened by
|
||||
evaluate.LINE_WINDOW, plus the declared sink line where the fix does not touch the sink (LearnPress).
|
||||
|
||||
The instrument must have read its input: every pin reports the PHP files and bytes the scan read, and a pin with
|
||||
zero PHP bytes fails the run.
|
||||
|
||||
Usage: python measure_php_quick.py <corpus dir> <out.json>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, "benchmarks/independent")
|
||||
import evaluate # noqa: E402
|
||||
|
||||
from openultrasast.findings import quick_scan_findings # noqa: E402
|
||||
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints # noqa: E402
|
||||
from openultrasast.preprocess import preprocess_repository # noqa: E402
|
||||
from openultrasast.rank import rank_targets # noqa: E402
|
||||
|
||||
V1 = ("wp-ultimate-member-sqli", "wp-learnpress-sqli", "phpmyfaq-captcha-ua-sqli", "yeswiki-bazar-form-sqli")
|
||||
DECLARED = {"wp-learnpress-sqli": ("inc/Databases/class-lp-db.php", "$FIELDS = implode( ',', array_unique( $filter->only_fields ) );")}
|
||||
W = evaluate.LINE_WINDOW
|
||||
|
||||
|
||||
def scan(root: Path) -> tuple[list, dict]:
|
||||
_, targets = preprocess_repository(root)
|
||||
php = [t for t in targets if t.language == "php"]
|
||||
read = {"php_files": len(php), "php_bytes": sum(len((root / t.path).read_bytes()) for t in php)}
|
||||
if not read["php_bytes"]:
|
||||
raise SystemExit(f"instrument failure: no PHP bytes read under {root}")
|
||||
targets = attach_reachability_hints(targets, analyze_entry_points(root, targets))
|
||||
started = time.monotonic()
|
||||
findings = [f for f in quick_scan_findings(root, targets, rank_targets(targets)) if f.finding_id.startswith("php-")]
|
||||
read["scan_seconds"] = round(time.monotonic() - started, 1)
|
||||
return findings, read
|
||||
|
||||
|
||||
def function_range(path: Path, name: str) -> tuple[int, int]:
|
||||
lines = path.read_text(errors="replace").splitlines()
|
||||
start = next(i for i, line in enumerate(lines, 1) if f"function {name}(" in line)
|
||||
end = next((i for i, line in enumerate(lines[start:], start + 1) if "function " in line), len(lines))
|
||||
return start, end - 1
|
||||
|
||||
|
||||
def line_of(path: Path, needle: str) -> int:
|
||||
return next(i for i, line in enumerate(path.read_text(errors="replace").splitlines(), 1) if needle in line)
|
||||
|
||||
|
||||
def at(finding, sites: dict[str, list[tuple[int, int]]]) -> bool:
|
||||
return any(first <= (finding.line or -1) <= last for first, last in sites.get(finding.path, []))
|
||||
|
||||
|
||||
def summarize(findings: list, sites: dict[str, list[tuple[int, int]]]) -> dict:
|
||||
hits = [f for f in findings if at(f, sites)]
|
||||
return {
|
||||
"findings": len(findings),
|
||||
"by_rule": dict(sorted(Counter(f.finding_id.split(":", 1)[0] for f in findings).items())),
|
||||
"by_rule_enabled": len([f for f in findings if f.status == "enabled"]),
|
||||
"at_known_site": [f.finding_id for f in hits],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
corpus, out = Path(sys.argv[1]), Path(sys.argv[2])
|
||||
checkouts = corpus / "checkouts"
|
||||
result: dict = {"window": W, "cases": {}}
|
||||
# pmpro: the declared CVE function on each pin.
|
||||
pmpro_dirs = {
|
||||
"vulnerable": Path.home() / ".cache/openultrasast/repos/pmpro/1a900be3f09f",
|
||||
"fixed": Path.home() / ".cache/openultrasast/repos/pmpro/2f65530f9442",
|
||||
"benign": checkouts / "pmpro" / "benign",
|
||||
}
|
||||
pmpro: dict = {}
|
||||
for label, root in pmpro_dirs.items():
|
||||
findings, read = scan(root)
|
||||
rel = "classes/class.memberorder.php"
|
||||
first, last = function_range(root / rel, "getMemberOrderByCode")
|
||||
sites = {rel: [(first, last)]}
|
||||
pmpro[label] = {"root": str(root), "read": read, "known_site": f"{rel}:{first}-{last}", **summarize(findings, sites)}
|
||||
pmpro[label]["_ids"] = sorted(f.finding_id for f in findings)
|
||||
print("pmpro", label, read, pmpro[label]["findings"], pmpro[label]["at_known_site"], file=sys.stderr)
|
||||
result["cases"]["pmpro"] = pmpro
|
||||
for case in evaluate.cases():
|
||||
if case["id"] not in V1:
|
||||
continue
|
||||
entry: dict = {}
|
||||
for label, side in (("vulnerable", "old"), ("fixed", "new")):
|
||||
root = checkouts / case["id"] / label
|
||||
findings, read = scan(root)
|
||||
sites = {
|
||||
path: [(a - W, b + W) for a, b in ranges]
|
||||
for path, ranges in evaluate.hunks(case, side).items()
|
||||
if path.endswith(".php") and not path.startswith(("tests/", "test/")) and "/tests/" not in path
|
||||
}
|
||||
if case["id"] in DECLARED:
|
||||
rel, needle = DECLARED[case["id"]]
|
||||
n = line_of(root / rel, needle)
|
||||
sites.setdefault(rel, []).append((n - W, n + W))
|
||||
entry[label] = {"read": read, "known_sites": {p: r for p, r in sorted(sites.items())}, **summarize(findings, sites)}
|
||||
entry[label]["_ids"] = sorted(f.finding_id for f in findings)
|
||||
print(case["id"], label, read, entry[label]["findings"], entry[label]["at_known_site"], file=sys.stderr)
|
||||
result["cases"][case["id"]] = entry
|
||||
result["fixtures"] = fixtures()
|
||||
result["rules"] = per_rule(result)
|
||||
for pins in result["cases"].values():
|
||||
for pin in pins.values():
|
||||
pin.pop("_ids")
|
||||
out.write_text(json.dumps(result, indent=1, sort_keys=True) + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
def fixtures() -> dict:
|
||||
"""The committed fixture benchmarks, scored exactly as `ousast benchmark` scores them."""
|
||||
from openultrasast.benchmark import BenchmarkRun, evaluate_benchmark, load_benchmark_manifest, resolve_benchmark_source
|
||||
from openultrasast.gate import _quick_scan
|
||||
|
||||
result = {}
|
||||
for name in ("php-vulnerable", "php-benign"):
|
||||
path = Path("benchmarks/manifests") / f"{name}.toml"
|
||||
manifest = load_benchmark_manifest(path)
|
||||
target = resolve_benchmark_source(path, manifest)
|
||||
findings = _quick_scan(target)
|
||||
run = BenchmarkRun(benchmark_run_id="php-quick", root=Path("/tmp/php-quick"), manifest=manifest)
|
||||
bench = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None)
|
||||
m = bench.metrics
|
||||
result[name] = {
|
||||
"expected": m.expected_findings_total, "matched": m.matched_findings_total, "findings": m.actual_findings_total,
|
||||
"false_positives": [fp.finding_id for fp in bench.false_positives], "missed": [x.rule_id or x.cwe for x in bench.misses],
|
||||
"true_positives": sorted({f.finding_id for f in findings} - {fp.finding_id for fp in bench.false_positives}),
|
||||
} # fmt: skip
|
||||
return result
|
||||
|
||||
|
||||
def per_rule(result: dict) -> dict:
|
||||
"""Firings per rule and the status/precision policy applied to them (see the module docstring of the ruleset).
|
||||
|
||||
enabled -- the rule hit a known vulnerable site on the corpus, or it never fired on the corpus (no false-alert
|
||||
pressure measured; fixture evidence only);
|
||||
shadow -- it fired on the corpus but never at a known site: every corpus firing was on code whose known bug is
|
||||
elsewhere, and it fired as often on the fixed pins.
|
||||
precision lower bound = (known-site hits on vulnerable pins + fixture true positives)
|
||||
/ (firings on vulnerable pins + fixture firings, both twins); an unlabeled firing counts false.
|
||||
"""
|
||||
rows: dict = defaultdict(lambda: defaultdict(int))
|
||||
for pins in result["cases"].values():
|
||||
for label, pin in pins.items():
|
||||
for fid in pin["_ids"]:
|
||||
rows[fid.split(":", 1)[0]][label] += 1
|
||||
for fid in pin["at_known_site"]:
|
||||
rows[fid.split(":", 1)[0]][label + "_at_site"] += 1
|
||||
for name, fixture in result["fixtures"].items():
|
||||
for fid in fixture["true_positives"]:
|
||||
rows[fid.split(":", 1)[0]]["fixture_tp"] += 1
|
||||
for fid in fixture["false_positives"]:
|
||||
rows[fid.split(":", 1)[0]]["fixture_fp"] += 1
|
||||
rules = {}
|
||||
for rule, row in sorted(rows.items()):
|
||||
fired, at_site = row["vulnerable"], row["vulnerable_at_site"]
|
||||
status = "enabled" if at_site or not fired else "shadow"
|
||||
denominator = fired + row["fixture_tp"] + row["fixture_fp"]
|
||||
precision = round((at_site + row["fixture_tp"]) / denominator, 3) if denominator else None
|
||||
rules[rule] = {**dict(sorted(row.items())), "status": status, "precision_lower_bound": precision, "n": denominator}
|
||||
return rules
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,864 @@
|
||||
{
|
||||
"cases": {
|
||||
"phpmyfaq-captcha-ua-sqli": {
|
||||
"fixed": {
|
||||
"at_known_site": [
|
||||
"php-sql-sprintf-unescaped:phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php:294",
|
||||
"php-sql-sprintf-unescaped:phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php:320",
|
||||
"php-sql-sprintf-unescaped:phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php:336"
|
||||
],
|
||||
"by_rule": {
|
||||
"php-sql-sprintf-unescaped": 44,
|
||||
"php-sql-unquoted-concat": 10,
|
||||
"php-sql-unquoted-interpolation": 5
|
||||
},
|
||||
"by_rule_enabled": 59,
|
||||
"findings": 59,
|
||||
"known_sites": {
|
||||
"phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php": [
|
||||
[
|
||||
267,
|
||||
291
|
||||
],
|
||||
[
|
||||
282,
|
||||
302
|
||||
],
|
||||
[
|
||||
291,
|
||||
313
|
||||
],
|
||||
[
|
||||
296,
|
||||
316
|
||||
],
|
||||
[
|
||||
304,
|
||||
329
|
||||
],
|
||||
[
|
||||
316,
|
||||
336
|
||||
],
|
||||
[
|
||||
318,
|
||||
338
|
||||
],
|
||||
[
|
||||
321,
|
||||
341
|
||||
],
|
||||
[
|
||||
334,
|
||||
357
|
||||
],
|
||||
[
|
||||
340,
|
||||
360
|
||||
],
|
||||
[
|
||||
475,
|
||||
499
|
||||
],
|
||||
[
|
||||
482,
|
||||
506
|
||||
]
|
||||
]
|
||||
},
|
||||
"read": {
|
||||
"php_bytes": 12893136,
|
||||
"php_files": 762,
|
||||
"scan_seconds": 14.4
|
||||
}
|
||||
},
|
||||
"vulnerable": {
|
||||
"at_known_site": [
|
||||
"php-sql-sprintf-unescaped:phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php:289",
|
||||
"php-sql-sprintf-unescaped:phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php:309",
|
||||
"php-sql-sprintf-unescaped:phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php:325",
|
||||
"php-sql-sprintf-unescaped:phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php:474"
|
||||
],
|
||||
"by_rule": {
|
||||
"php-sql-sprintf-unescaped": 45,
|
||||
"php-sql-unquoted-concat": 10,
|
||||
"php-sql-unquoted-interpolation": 5
|
||||
},
|
||||
"by_rule_enabled": 60,
|
||||
"findings": 60,
|
||||
"known_sites": {
|
||||
"phpmyfaq/src/phpMyFAQ/Captcha/BuiltinCaptcha.php": [
|
||||
[
|
||||
266,
|
||||
286
|
||||
],
|
||||
[
|
||||
277,
|
||||
297
|
||||
],
|
||||
[
|
||||
286,
|
||||
308
|
||||
],
|
||||
[
|
||||
291,
|
||||
311
|
||||
],
|
||||
[
|
||||
298,
|
||||
318
|
||||
],
|
||||
[
|
||||
305,
|
||||
325
|
||||
],
|
||||
[
|
||||
307,
|
||||
327
|
||||
],
|
||||
[
|
||||
310,
|
||||
330
|
||||
],
|
||||
[
|
||||
323,
|
||||
346
|
||||
],
|
||||
[
|
||||
329,
|
||||
349
|
||||
],
|
||||
[
|
||||
464,
|
||||
484
|
||||
],
|
||||
[
|
||||
466,
|
||||
486
|
||||
]
|
||||
]
|
||||
},
|
||||
"read": {
|
||||
"php_bytes": 12888581,
|
||||
"php_files": 762,
|
||||
"scan_seconds": 13.9
|
||||
}
|
||||
}
|
||||
},
|
||||
"pmpro": {
|
||||
"benign": {
|
||||
"at_known_site": [
|
||||
"php-wpdb-sql-composition:classes/class.memberorder.php:937"
|
||||
],
|
||||
"by_rule": {
|
||||
"php-sql-unquoted-concat": 9,
|
||||
"php-sql-unquoted-interpolation": 28,
|
||||
"php-wpdb-sql-composition": 55
|
||||
},
|
||||
"by_rule_enabled": 92,
|
||||
"findings": 92,
|
||||
"known_site": "classes/class.memberorder.php:934-946",
|
||||
"read": {
|
||||
"php_bytes": 4244314,
|
||||
"php_files": 637,
|
||||
"scan_seconds": 5.5
|
||||
},
|
||||
"root": "/tmp/claude-1000/-home-mc-Source-OpenUltraSAST/c09e30b2-39a4-4564-bc31-6ea71ffaa198/scratchpad/phpquick/corpus/checkouts/pmpro/benign"
|
||||
},
|
||||
"fixed": {
|
||||
"at_known_site": [],
|
||||
"by_rule": {
|
||||
"php-sql-unquoted-concat": 6,
|
||||
"php-sql-unquoted-interpolation": 21,
|
||||
"php-wpdb-sql-composition": 39
|
||||
},
|
||||
"by_rule_enabled": 66,
|
||||
"findings": 66,
|
||||
"known_site": "classes/class.memberorder.php:934-946",
|
||||
"read": {
|
||||
"php_bytes": 4246202,
|
||||
"php_files": 637,
|
||||
"scan_seconds": 5.3
|
||||
},
|
||||
"root": "/home/mc/.cache/openultrasast/repos/pmpro/2f65530f9442"
|
||||
},
|
||||
"vulnerable": {
|
||||
"at_known_site": [
|
||||
"php-wpdb-sql-composition:classes/class.memberorder.php:936"
|
||||
],
|
||||
"by_rule": {
|
||||
"php-sql-unquoted-concat": 9,
|
||||
"php-sql-unquoted-interpolation": 28,
|
||||
"php-wpdb-sql-composition": 55
|
||||
},
|
||||
"by_rule_enabled": 92,
|
||||
"findings": 92,
|
||||
"known_site": "classes/class.memberorder.php:933-945",
|
||||
"read": {
|
||||
"php_bytes": 4244270,
|
||||
"php_files": 637,
|
||||
"scan_seconds": 5.3
|
||||
},
|
||||
"root": "/home/mc/.cache/openultrasast/repos/pmpro/1a900be3f09f"
|
||||
}
|
||||
},
|
||||
"wp-learnpress-sqli": {
|
||||
"fixed": {
|
||||
"at_known_site": [],
|
||||
"by_rule": {
|
||||
"php-sql-unquoted-concat": 19,
|
||||
"php-sql-unquoted-interpolation": 23,
|
||||
"php-unserialize": 1,
|
||||
"php-wpdb-sql-composition": 4
|
||||
},
|
||||
"by_rule_enabled": 46,
|
||||
"findings": 47,
|
||||
"known_sites": {
|
||||
"inc/Databases/class-lp-db.php": [
|
||||
[
|
||||
597,
|
||||
617
|
||||
]
|
||||
],
|
||||
"inc/Models/Courses.php": [
|
||||
[
|
||||
9,
|
||||
29
|
||||
],
|
||||
[
|
||||
72,
|
||||
94
|
||||
],
|
||||
[
|
||||
78,
|
||||
107
|
||||
]
|
||||
],
|
||||
"inc/class-lp-assets.php": [
|
||||
[
|
||||
299,
|
||||
319
|
||||
]
|
||||
],
|
||||
"inc/jwt/rest-api/version1/class-lp-rest-courses-v1-controller.php": [
|
||||
[
|
||||
425,
|
||||
445
|
||||
],
|
||||
[
|
||||
431,
|
||||
459
|
||||
],
|
||||
[
|
||||
448,
|
||||
468
|
||||
],
|
||||
[
|
||||
532,
|
||||
552
|
||||
],
|
||||
[
|
||||
536,
|
||||
557
|
||||
]
|
||||
]
|
||||
},
|
||||
"read": {
|
||||
"php_bytes": 3515681,
|
||||
"php_files": 801,
|
||||
"scan_seconds": 4.4
|
||||
}
|
||||
},
|
||||
"vulnerable": {
|
||||
"at_known_site": [],
|
||||
"by_rule": {
|
||||
"php-sql-unquoted-concat": 19,
|
||||
"php-sql-unquoted-interpolation": 23,
|
||||
"php-unserialize": 1,
|
||||
"php-wpdb-sql-composition": 4
|
||||
},
|
||||
"by_rule_enabled": 46,
|
||||
"findings": 47,
|
||||
"known_sites": {
|
||||
"inc/Databases/class-lp-db.php": [
|
||||
[
|
||||
597,
|
||||
617
|
||||
]
|
||||
],
|
||||
"inc/Models/Courses.php": [
|
||||
[
|
||||
8,
|
||||
28
|
||||
],
|
||||
[
|
||||
70,
|
||||
90
|
||||
],
|
||||
[
|
||||
73,
|
||||
93
|
||||
]
|
||||
],
|
||||
"inc/class-lp-assets.php": [
|
||||
[
|
||||
299,
|
||||
319
|
||||
]
|
||||
],
|
||||
"inc/jwt/rest-api/version1/class-lp-rest-courses-v1-controller.php": [
|
||||
[
|
||||
425,
|
||||
445
|
||||
],
|
||||
[
|
||||
430,
|
||||
450
|
||||
],
|
||||
[
|
||||
440,
|
||||
462
|
||||
],
|
||||
[
|
||||
526,
|
||||
547
|
||||
],
|
||||
[
|
||||
531,
|
||||
552
|
||||
]
|
||||
]
|
||||
},
|
||||
"read": {
|
||||
"php_bytes": 3515194,
|
||||
"php_files": 801,
|
||||
"scan_seconds": 4.7
|
||||
}
|
||||
}
|
||||
},
|
||||
"wp-ultimate-member-sqli": {
|
||||
"fixed": {
|
||||
"at_known_site": [
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:793",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:810",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:816",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:825",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:833",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:842",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:902",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:907"
|
||||
],
|
||||
"by_rule": {
|
||||
"php-sql-unquoted-concat": 2,
|
||||
"php-sql-unquoted-interpolation": 9,
|
||||
"php-wpdb-sql-composition": 3
|
||||
},
|
||||
"by_rule_enabled": 14,
|
||||
"findings": 14,
|
||||
"known_sites": {
|
||||
"includes/core/class-member-directory-meta.php": [
|
||||
[
|
||||
778,
|
||||
799
|
||||
],
|
||||
[
|
||||
780,
|
||||
800
|
||||
],
|
||||
[
|
||||
782,
|
||||
802
|
||||
],
|
||||
[
|
||||
798,
|
||||
818
|
||||
],
|
||||
[
|
||||
805,
|
||||
825
|
||||
],
|
||||
[
|
||||
814,
|
||||
834
|
||||
],
|
||||
[
|
||||
822,
|
||||
842
|
||||
],
|
||||
[
|
||||
831,
|
||||
851
|
||||
],
|
||||
[
|
||||
862,
|
||||
882
|
||||
],
|
||||
[
|
||||
891,
|
||||
911
|
||||
],
|
||||
[
|
||||
893,
|
||||
913
|
||||
],
|
||||
[
|
||||
896,
|
||||
916
|
||||
]
|
||||
]
|
||||
},
|
||||
"read": {
|
||||
"php_bytes": 2588115,
|
||||
"php_files": 261,
|
||||
"scan_seconds": 3.5
|
||||
}
|
||||
},
|
||||
"vulnerable": {
|
||||
"at_known_site": [
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:791",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:807",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:812",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:820",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:827",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:835",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:894",
|
||||
"php-sql-unquoted-interpolation:includes/core/class-member-directory-meta.php:898"
|
||||
],
|
||||
"by_rule": {
|
||||
"php-sql-unquoted-concat": 2,
|
||||
"php-sql-unquoted-interpolation": 9,
|
||||
"php-wpdb-sql-composition": 3
|
||||
},
|
||||
"by_rule_enabled": 14,
|
||||
"findings": 14,
|
||||
"known_sites": {
|
||||
"includes/core/class-member-directory-meta.php": [
|
||||
[
|
||||
777,
|
||||
797
|
||||
],
|
||||
[
|
||||
779,
|
||||
799
|
||||
],
|
||||
[
|
||||
780,
|
||||
800
|
||||
],
|
||||
[
|
||||
795,
|
||||
815
|
||||
],
|
||||
[
|
||||
801,
|
||||
821
|
||||
],
|
||||
[
|
||||
809,
|
||||
829
|
||||
],
|
||||
[
|
||||
816,
|
||||
836
|
||||
],
|
||||
[
|
||||
824,
|
||||
844
|
||||
],
|
||||
[
|
||||
855,
|
||||
875
|
||||
],
|
||||
[
|
||||
883,
|
||||
903
|
||||
],
|
||||
[
|
||||
885,
|
||||
905
|
||||
],
|
||||
[
|
||||
887,
|
||||
907
|
||||
]
|
||||
]
|
||||
},
|
||||
"read": {
|
||||
"php_bytes": 2587061,
|
||||
"php_files": 261,
|
||||
"scan_seconds": 3.3
|
||||
}
|
||||
}
|
||||
},
|
||||
"yeswiki-bazar-form-sqli": {
|
||||
"fixed": {
|
||||
"at_known_site": [],
|
||||
"by_rule": {
|
||||
"php-code-eval": 2,
|
||||
"php-sql-call-composition": 3,
|
||||
"php-sql-unquoted-concat": 15,
|
||||
"php-sql-unquoted-interpolation": 4,
|
||||
"php-unserialize": 2,
|
||||
"php-xss-echo-input": 2
|
||||
},
|
||||
"by_rule_enabled": 19,
|
||||
"findings": 28,
|
||||
"known_sites": {
|
||||
"tools/bazar/controllers/FormController.php": [
|
||||
[
|
||||
29,
|
||||
51
|
||||
]
|
||||
],
|
||||
"tools/bazar/services/FormManager.php": [
|
||||
[
|
||||
248,
|
||||
268
|
||||
]
|
||||
]
|
||||
},
|
||||
"read": {
|
||||
"php_bytes": 3971093,
|
||||
"php_files": 655,
|
||||
"scan_seconds": 5.2
|
||||
}
|
||||
},
|
||||
"vulnerable": {
|
||||
"at_known_site": [
|
||||
"php-sql-unquoted-concat:tools/bazar/services/FormManager.php:258"
|
||||
],
|
||||
"by_rule": {
|
||||
"php-code-eval": 2,
|
||||
"php-sql-call-composition": 3,
|
||||
"php-sql-unquoted-concat": 16,
|
||||
"php-sql-unquoted-interpolation": 4,
|
||||
"php-unserialize": 2,
|
||||
"php-xss-echo-input": 2
|
||||
},
|
||||
"by_rule_enabled": 20,
|
||||
"findings": 29,
|
||||
"known_sites": {
|
||||
"tools/bazar/controllers/FormController.php": [
|
||||
[
|
||||
28,
|
||||
48
|
||||
]
|
||||
],
|
||||
"tools/bazar/services/FormManager.php": [
|
||||
[
|
||||
248,
|
||||
268
|
||||
]
|
||||
]
|
||||
},
|
||||
"read": {
|
||||
"php_bytes": 3970854,
|
||||
"php_files": 655,
|
||||
"scan_seconds": 5.1
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"engine": {
|
||||
"pmpro": {
|
||||
"fixed": {
|
||||
"at_known_site": [],
|
||||
"completed": 318,
|
||||
"degradations": [
|
||||
"cross_partition_semantics_unresolved",
|
||||
"partition_file_census_incomplete",
|
||||
"process_timeout",
|
||||
"query_budget_reserved"
|
||||
],
|
||||
"findings": 6,
|
||||
"questions": 382,
|
||||
"read": "read 673 files, 4603062 bytes; 4461 regions offer ('injection',)",
|
||||
"seconds": 2025.8,
|
||||
"sites": [
|
||||
"classes/class-pmpro-discount-codes.php:266:save",
|
||||
"classes/class.memberorder.php:1469:saveOrder",
|
||||
"classes/class.pmproemail.php:549:sendCheckoutEmail",
|
||||
"classes/class.pmproemail.php:668:sendCheckoutAdminEmail",
|
||||
"includes/functions.php:3823:pmpro_insert_or_replace",
|
||||
"services/ipnhandler.php:819:pmpro_ipnSaveOrder"
|
||||
]
|
||||
},
|
||||
"source": "frozen export of 45d02ec, openultrasast:dev, --memory 3g, --network none, family injection",
|
||||
"vulnerable": {
|
||||
"at_known_site": [
|
||||
"classes/class.memberorder.php:936:getMemberOrderByCode"
|
||||
],
|
||||
"completed": 125,
|
||||
"degradations": [
|
||||
"cross_partition_semantics_unresolved",
|
||||
"partition_file_census_incomplete",
|
||||
"process_timeout",
|
||||
"query_budget_reserved"
|
||||
],
|
||||
"findings": 10,
|
||||
"questions": 382,
|
||||
"read": "read 673 files, 4601130 bytes; 4461 regions offer ('injection',)",
|
||||
"seconds": 1993.4,
|
||||
"sites": [
|
||||
"classes/class.memberorder.php:1467:saveOrder",
|
||||
"classes/class.memberorder.php:925:getLastMemberOrder",
|
||||
"classes/class.memberorder.php:936:getMemberOrderByCode",
|
||||
"classes/class.memberorder.php:984:getMemberOrderByPayPalToken",
|
||||
"classes/class.pmproemail.php:550:sendCheckoutEmail",
|
||||
"classes/class.pmproemail.php:661:sendCheckoutAdminEmail",
|
||||
"includes/profile.php:337:pmpro_membership_level_profile_fields_update",
|
||||
"includes/profile.php:344:pmpro_membership_level_profile_fields_update",
|
||||
"includes/profile.php:350:pmpro_membership_level_profile_fields_update",
|
||||
"services/ipnhandler.php:819:pmpro_ipnSaveOrder"
|
||||
]
|
||||
}
|
||||
},
|
||||
"v1_recorded": {
|
||||
"phpmyfaq-captcha-ua-sqli": {
|
||||
"detected": false,
|
||||
"fixed": {
|
||||
"completed": 0,
|
||||
"findings": 0,
|
||||
"questions": 5000,
|
||||
"seconds": 426.2
|
||||
},
|
||||
"unanswered": true,
|
||||
"vulnerable": {
|
||||
"completed": 0,
|
||||
"findings": 0,
|
||||
"questions": 5000,
|
||||
"seconds": 437.7
|
||||
}
|
||||
},
|
||||
"source": "not re-run: the population-v1 run of 2026-09-24 (analyzer ebcc2cf), ~/ousast-results/independent-v1/score.json",
|
||||
"wp-learnpress-sqli": {
|
||||
"detected": false,
|
||||
"fixed": {
|
||||
"completed": 4815,
|
||||
"findings": 0,
|
||||
"questions": 5000,
|
||||
"seconds": 470.5
|
||||
},
|
||||
"unanswered": false,
|
||||
"vulnerable": {
|
||||
"completed": 4815,
|
||||
"findings": 0,
|
||||
"questions": 5000,
|
||||
"seconds": 414.2
|
||||
}
|
||||
},
|
||||
"wp-ultimate-member-sqli": {
|
||||
"detected": false,
|
||||
"fixed": {
|
||||
"completed": 76,
|
||||
"findings": 0,
|
||||
"questions": 303,
|
||||
"seconds": 218.0
|
||||
},
|
||||
"unanswered": false,
|
||||
"vulnerable": {
|
||||
"completed": 76,
|
||||
"findings": 0,
|
||||
"questions": 303,
|
||||
"seconds": 215.8
|
||||
}
|
||||
},
|
||||
"yeswiki-bazar-form-sqli": {
|
||||
"detected": false,
|
||||
"fixed": {
|
||||
"completed": 0,
|
||||
"findings": 4,
|
||||
"questions": 1128,
|
||||
"seconds": 306.9
|
||||
},
|
||||
"unanswered": true,
|
||||
"vulnerable": {
|
||||
"completed": 0,
|
||||
"findings": 4,
|
||||
"questions": 1128,
|
||||
"seconds": 334.9
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"fixtures": {
|
||||
"php-benign": {
|
||||
"expected": 0,
|
||||
"false_positives": [
|
||||
"php-file-inclusion-input:app.php:64",
|
||||
"php-sql-unquoted-concat:app.php:87"
|
||||
],
|
||||
"findings": 2,
|
||||
"matched": 0,
|
||||
"missed": [],
|
||||
"true_positives": []
|
||||
},
|
||||
"php-vulnerable": {
|
||||
"expected": 14,
|
||||
"false_positives": [],
|
||||
"findings": 13,
|
||||
"matched": 13,
|
||||
"missed": [
|
||||
"CWE-89"
|
||||
],
|
||||
"true_positives": [
|
||||
"php-code-eval:app.php:46",
|
||||
"php-command-injection:app.php:41",
|
||||
"php-file-inclusion-input:app.php:62",
|
||||
"php-open-redirect:app.php:78",
|
||||
"php-path-input:app.php:67",
|
||||
"php-sql-call-composition:app.php:16",
|
||||
"php-sql-sprintf-unescaped:app.php:32",
|
||||
"php-sql-unquoted-concat:app.php:27",
|
||||
"php-sql-unquoted-interpolation:app.php:22",
|
||||
"php-ssrf-input:app.php:72",
|
||||
"php-unserialize:app.php:52",
|
||||
"php-wpdb-sql-composition:app.php:11",
|
||||
"php-xss-echo-input:app.php:57"
|
||||
]
|
||||
}
|
||||
},
|
||||
"provenance": {
|
||||
"analyzer_base_commit": "45d02ec",
|
||||
"corpus": "development only: benchmarks/push/inputs.json pmpro vulnerable/fixed/benign (role development) and the spent population-v1 PHP cases' vulnerable/fixed pins; population-v2 was not read",
|
||||
"date": "2026-09-30",
|
||||
"harness": "measure_php_quick.py (this directory)",
|
||||
"in_sample": "the rules were written and tightened while looking at these pins and the fixtures, so every number here is in-sample (development), not a holdout result",
|
||||
"instrument": "every pin reports php_files and php_bytes read by the scan; a zero-byte pin fails the harness",
|
||||
"known_site": "pmpro: getMemberOrderByCode on each pin; v1: fix hunks (vulnerable side / fixed side, test paths dropped) widened by 10 lines, plus the declared LearnPress sink line"
|
||||
},
|
||||
"rules": {
|
||||
"php-code-eval": {
|
||||
"fixed": 2,
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 3,
|
||||
"precision_lower_bound": 0.333,
|
||||
"status": "shadow",
|
||||
"vulnerable": 2,
|
||||
"vulnerable_at_site": 0
|
||||
},
|
||||
"php-command-injection": {
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 1,
|
||||
"precision_lower_bound": 1.0,
|
||||
"status": "enabled",
|
||||
"vulnerable": 0,
|
||||
"vulnerable_at_site": 0
|
||||
},
|
||||
"php-file-inclusion-input": {
|
||||
"fixture_fp": 1,
|
||||
"fixture_tp": 1,
|
||||
"n": 2,
|
||||
"precision_lower_bound": 0.5,
|
||||
"status": "enabled",
|
||||
"vulnerable": 0,
|
||||
"vulnerable_at_site": 0
|
||||
},
|
||||
"php-open-redirect": {
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 1,
|
||||
"precision_lower_bound": 1.0,
|
||||
"status": "enabled",
|
||||
"vulnerable": 0,
|
||||
"vulnerable_at_site": 0
|
||||
},
|
||||
"php-path-input": {
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 1,
|
||||
"precision_lower_bound": 1.0,
|
||||
"status": "enabled",
|
||||
"vulnerable": 0,
|
||||
"vulnerable_at_site": 0
|
||||
},
|
||||
"php-sql-call-composition": {
|
||||
"fixed": 3,
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 4,
|
||||
"precision_lower_bound": 0.25,
|
||||
"status": "shadow",
|
||||
"vulnerable": 3,
|
||||
"vulnerable_at_site": 0
|
||||
},
|
||||
"php-sql-sprintf-unescaped": {
|
||||
"fixed": 44,
|
||||
"fixed_at_site": 3,
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 46,
|
||||
"precision_lower_bound": 0.109,
|
||||
"status": "enabled",
|
||||
"vulnerable": 45,
|
||||
"vulnerable_at_site": 4
|
||||
},
|
||||
"php-sql-unquoted-concat": {
|
||||
"benign": 9,
|
||||
"fixed": 52,
|
||||
"fixture_fp": 1,
|
||||
"fixture_tp": 1,
|
||||
"n": 58,
|
||||
"precision_lower_bound": 0.034,
|
||||
"status": "enabled",
|
||||
"vulnerable": 56,
|
||||
"vulnerable_at_site": 1
|
||||
},
|
||||
"php-sql-unquoted-interpolation": {
|
||||
"benign": 28,
|
||||
"fixed": 62,
|
||||
"fixed_at_site": 8,
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 70,
|
||||
"precision_lower_bound": 0.129,
|
||||
"status": "enabled",
|
||||
"vulnerable": 69,
|
||||
"vulnerable_at_site": 8
|
||||
},
|
||||
"php-ssrf-input": {
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 1,
|
||||
"precision_lower_bound": 1.0,
|
||||
"status": "enabled",
|
||||
"vulnerable": 0,
|
||||
"vulnerable_at_site": 0
|
||||
},
|
||||
"php-unserialize": {
|
||||
"fixed": 3,
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 4,
|
||||
"precision_lower_bound": 0.25,
|
||||
"status": "shadow",
|
||||
"vulnerable": 3,
|
||||
"vulnerable_at_site": 0
|
||||
},
|
||||
"php-wpdb-sql-composition": {
|
||||
"benign": 55,
|
||||
"benign_at_site": 1,
|
||||
"fixed": 46,
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 63,
|
||||
"precision_lower_bound": 0.032,
|
||||
"status": "enabled",
|
||||
"vulnerable": 62,
|
||||
"vulnerable_at_site": 1
|
||||
},
|
||||
"php-xss-echo-input": {
|
||||
"fixed": 2,
|
||||
"fixture_fp": 0,
|
||||
"fixture_tp": 1,
|
||||
"n": 3,
|
||||
"precision_lower_bound": 0.333,
|
||||
"status": "shadow",
|
||||
"vulnerable": 2,
|
||||
"vulnerable_at_site": 0
|
||||
}
|
||||
},
|
||||
"window": 10
|
||||
}
|
||||
@@ -275,7 +275,7 @@ def _main(argv: list[str] | None) -> int:
|
||||
plane_remember.add_argument("--population", help="the population the run measured (default: the Run's annotation)")
|
||||
plane_remember.add_argument("--split", help="the split of that population (default: the Run's annotation)")
|
||||
plane_engine = plane_sub.add_parser(
|
||||
"alerts-engine", help="a Run's `alerts` for languages quick mode does not cover (PHP), from the engine image on this host"
|
||||
"alerts-engine", help="a Run's `alerts` for PHP and languages quick mode does not cover, from the engine image on this host"
|
||||
)
|
||||
plane_engine.add_argument("run_manifest", type=Path)
|
||||
plane_engine.add_argument(
|
||||
|
||||
@@ -196,6 +196,8 @@ def _match_is_in_comment(text: str, offset: int, language: str = "") -> bool:
|
||||
return True
|
||||
comment_column = line.find("#")
|
||||
return comment_column != -1 and comment_column < column
|
||||
if language == "php" and stripped.startswith("#") and not stripped.startswith("#["):
|
||||
return True # PHP also has `#` line comments; `#[...]` is a PHP 8 attribute, not a comment
|
||||
if stripped.startswith(("//", "/*", "*")):
|
||||
return True
|
||||
comment_column = line.find("//")
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
"""Engine alerts for the languages quick mode does not cover, produced on the host (harnessx-removal Req 6.3).
|
||||
"""Engine alerts for the languages quick mode does not cover, and for PHP beside its quick rules
|
||||
(``ENGINE_ALONGSIDE_QUICK``), produced on the host (harnessx-removal Req 6.3).
|
||||
|
||||
The loop's ``alerts`` task runs the quick-mode scan, which has rules only for the languages its ruleset declares; a
|
||||
PHP case yields zero alerts that mean nothing (``coverage: none``). PHP detection comes from the Joern taint engine
|
||||
(``ruleset/semantic/php.toml``), which runs in the ``openultrasast:dev`` image, not in the runner sandbox. So
|
||||
``ousast plane alerts-engine <Run.yaml>`` does the ``alerts`` task's work on the host for every case whose files
|
||||
include a language quick mode does not cover but the engine does:
|
||||
case in any other language yields zero alerts that mean nothing (``coverage: none``). PHP has quick rules since
|
||||
2026-09-30 (``ruleset/php/rules.toml``, line patterns over the engine's vocabulary), but its detection of record is
|
||||
still the Joern taint engine (``ruleset/semantic/php.toml``), which runs in the ``openultrasast:dev`` image, not in
|
||||
the runner sandbox. So ``ousast plane alerts-engine <Run.yaml>`` does the ``alerts`` task's work on the host for
|
||||
every case whose files include a language the engine models and quick mode does not cover, or PHP:
|
||||
|
||||
- export the case's vulnerable and fixed pins from the case cache (``git archive``, as
|
||||
``benchmarks/independent/evaluate.py`` does), run the quick scan on each (:func:`.tasks.alerts.scan`) and the
|
||||
@@ -42,6 +44,10 @@ IMAGE = "openultrasast:dev"
|
||||
MEMORY = "3g"
|
||||
DEADLINE = 1800.0
|
||||
GRACE = 600.0 # the container's own startup and teardown beyond the scan deadline
|
||||
# Languages whose alerts of record come from the engine even though quick mode has rules for them. PHP's quick rules
|
||||
# (ruleset/php, 2026-09-30) are a line layer with a low measured precision lower bound on the development corpus, so
|
||||
# the engine keeps running for PHP beside them, and the coverage reports the engine for it.
|
||||
ENGINE_ALONGSIDE_QUICK = frozenset({"php"})
|
||||
CACHE = Path.home() / ".cache" / "openultrasast" / "independent"
|
||||
DUMP = Path("benchmarks") / "push" / "finding_dump.py"
|
||||
_READ = re.compile(r"^read (\d+) files, (\d+) bytes", re.MULTILINE)
|
||||
@@ -193,7 +199,10 @@ def produce(
|
||||
files[role] = read.pop("languages")
|
||||
if read["files"] == 0:
|
||||
raise EngineAlertsError(f"{case.case_id} {role}: the quick scan read no file of {checkout}")
|
||||
needed |= {lang for lang in files[role] if lang != "unknown" and not covers(quick, lang) and lang in engine}
|
||||
needed |= {
|
||||
lang for lang in files[role]
|
||||
if lang != "unknown" and lang in engine and (not covers(quick, lang) or lang in ENGINE_ALONGSIDE_QUICK)
|
||||
} # fmt: skip
|
||||
if not needed:
|
||||
return {"case": case.case_id, "status": "skipped", "reason": "quick mode covers every language read", "languages": files}
|
||||
out_dir = work / "out"
|
||||
@@ -216,7 +225,7 @@ def produce(
|
||||
summary["in_fix_range"][role] = sum(1 for r in found if r["in_fix_range"] is True)
|
||||
keys = ("files", "bytes", "questions", "completed", "seconds", "degradations", "exit")
|
||||
summary["engine"][role] = {**{k: record.get(k) for k in keys}, "findings": len(record.get("findings") or [])}
|
||||
cov = coverage(files, quick, needed)
|
||||
cov = coverage(files, set(quick) - needed, needed) # a language the engine ran for is reported as the engine's
|
||||
summary.update(
|
||||
status="done", units_done=len(ROLES), units_total=len(ROLES), usd=0, calls=0, usage={}, model=None, source="engine",
|
||||
host=True, family=case.family, image=_image_id(image, runner), deadline=deadline, quick_languages=sorted(quick),
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
# PHP quick-mode rules: line/statement patterns over the engine's PHP vocabulary (semantic/php.toml).
|
||||
#
|
||||
# The engine (Joern taint, standard/deep) owns PHP data flow. These rules are the quick layer: they see one line
|
||||
# or one statement, so each one pairs a sink family from semantic/php.toml with the shape that makes it dangerous
|
||||
# WITHOUT flow -- a request superglobal on the sink's own line, or a query literal composed with a variable -- and
|
||||
# excludes the lines that carry that family's sanitizer. Sinks, sources and sanitizers are the engine's words:
|
||||
# sources $_GET $_POST $_REQUEST $_COOKIE ($_SERVER, $_FILES, php://input where the family reads them)
|
||||
# sanitizers $wpdb->prepare, the prepared_statement shape, sql_escape (quoted positions ONLY -- quoted_only),
|
||||
# coerce (intval/absint/floatval), html_escape, shell_escape, path_confine (basename/realpath)
|
||||
#
|
||||
# Measured on the PHP development corpus only (benchmarks/measurements/2026-09-30-php-quick-rules/): pmpro
|
||||
# introduce/repair/benign and the spent population-v1 PHP cases -- never population-v2. `status` and
|
||||
# `precision_estimate` come from that measurement: a rule whose firings on the corpus were not at a known site
|
||||
# is `shadow`, and precision_estimate is the fraction of corpus + fixture firings that were at a labeled site
|
||||
# (a lower bound: an unlabeled firing is counted false).
|
||||
|
||||
# --- SQL (sql, wpdb) ---------------------------------------------------------------------------------------
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-wpdb-sql-composition"
|
||||
title = "WordPress $wpdb query composed from a variable without $wpdb->prepare"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-89"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.032
|
||||
version = "1"
|
||||
pattern = '''\$wpdb->(get_var|get_results|get_row|get_col|query|replace)\s*\(\s*(?!\$wpdb->prepare)(?:"[^"\n]*\{?\$(?!wpdb->)[A-Za-z_]|[^\n;]*?["']\s*\.\s*\$(?!wpdb->)[A-Za-z_]|[^\n;]*?\$_(?:GET|POST|REQUEST|COOKIE)\b)'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-sql-call-composition"
|
||||
title = "PHP SQL call composed from a variable or request input"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-89"
|
||||
tags = ["injection"]
|
||||
status = "shadow"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.25
|
||||
version = "1"
|
||||
pattern = '''(?<![\w>:$\\])(mysqli_query|mysqli_multi_query|mysqli_real_query|mysql_query|pg_query|pg_send_query|sqlite_query|sqlite_unbuffered_query)\s*\((?:\s*"[^"\n]*\{?\$[A-Za-z_]|[^,\n;]*,\s*"[^"\n]*\{?\$[A-Za-z_]|[^\n;]*?["']\s*\.\s*\$[A-Za-z_]|[^\n;]*?\$_(?:GET|POST|REQUEST|COOKIE)\b)'''
|
||||
|
||||
# A variable in an UNQUOTED SQL position -- a number, an identifier, ORDER BY / LIMIT / IN (...) / VALUES (...).
|
||||
# The engine's sql_escape family is quoted_only: esc_sql and mysqli_real_escape_string protect a value inside a
|
||||
# quoted literal and nothing else, so an escaped value here is still injectable (Ultimate Member CVE-2024-1071,
|
||||
# YesWiki CVE-2026-46670). Coercion (intval/absint/(int)) is the fix, and a coerced value does not match.
|
||||
[[rule]]
|
||||
rule_id = "php-sql-unquoted-concat"
|
||||
title = "PHP SQL literal concatenates a variable into an unquoted position"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-89"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.034
|
||||
version = "1"
|
||||
pattern = '''(?<![$\w>])(?=(VALUES|IN|ORDER|GROUP|LIMIT|OFFSET|WHERE|AND|OR|SET|HAVING)\b)(?:(?:VALUES|IN)\s*\(|(?:ORDER|GROUP)\s+BY\b[^"'\n]*|(?:LIMIT|OFFSET)\s+(?:\d+\s*,\s*)?|(?:WHERE|AND|OR|SET|HAVING)\s+[\w.`]+\s*(?:=|<>|!=|<=|>=|<|>|LIKE)\s*)["']\s*\.\s*\$(?!wpdb->)(?!\w*(?:format|placeholder))[A-Za-z_]'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-sql-unquoted-interpolation"
|
||||
title = "PHP SQL string interpolates a variable into an unquoted position"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-89"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.129
|
||||
version = "1"
|
||||
pattern = '''(?<![$\w>])(?=(VALUES|IN|ORDER|GROUP|LIMIT|OFFSET|WHERE|AND|OR|SET|HAVING)\b)(?:(?:VALUES|IN)\s*\(\s*|(?:ORDER|GROUP)\s+BY\b[^"'\n;]*?|(?:LIMIT|OFFSET)\s+(?:\d+\s*,\s*)?|(?:WHERE|AND|OR|SET|HAVING)\s+[\w.`]+\s*(?:=|<>|!=|<=|>=|<|>|LIKE)\s*)\{?\$(?!wpdb->)(?!\w*(?:format|placeholder))[A-Za-z_]'''
|
||||
|
||||
# sprintf-built SQL with a quoted %s and no escaping or coercion anywhere in the statement (phpMyFAQ
|
||||
# CVE-2026-46364). Statement-scoped, so it spans lines up to the closing `);`.
|
||||
[[rule]]
|
||||
rule_id = "php-sql-sprintf-unescaped"
|
||||
title = "PHP sprintf-built SQL with an unescaped %s value"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-89"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.109
|
||||
version = "1"
|
||||
pattern = '''(sprintf)\(\s*["']\s*(?:SELECT|INSERT|UPDATE|DELETE|REPLACE)\b(?=(?:(?!\);)[\s\S]){0,2000}?'%s')(?=(?:(?![Ee]scape|quote\(|intval\(|absint\(|\(int\)|esc_sql\()[\s\S]){0,2000}?\);)'''
|
||||
|
||||
# --- command / code (command, code_eval) -------------------------------------------------------------------
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-command-injection"
|
||||
title = "PHP shell command built from a variable without escapeshellarg"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-78"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 1.0
|
||||
version = "1"
|
||||
pattern = '''(?<![\w>:$\\])(?<!function )(system|exec|shell_exec|passthru|popen|proc_open|pcntl_exec)\s*\((?![^\n]*escapeshell(?:arg|cmd)\s*\()[^,\n;]*\$[A-Za-z_]'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-code-eval"
|
||||
title = "PHP code evaluation of a variable or request input"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-95"
|
||||
tags = ["syscall_entry"]
|
||||
status = "shadow"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.333
|
||||
version = "1"
|
||||
pattern = '''(?<![\w>:$\\])(?<!function )(?=(eval|create_function|assert)\b)(?:(?:eval|create_function)\s*\([^\n;]*\$[A-Za-z_]|assert\s*\([^\n;]*\$_(?:GET|POST|REQUEST|COOKIE)\b)'''
|
||||
|
||||
# --- deserialization (unserialize) -------------------------------------------------------------------------
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-unserialize"
|
||||
title = "PHP unserialize of a variable or request input without allowed_classes"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-502"
|
||||
tags = ["deserialization"]
|
||||
status = "shadow"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.25
|
||||
version = "1"
|
||||
# `maybe_unserialize` (WordPress) runs on every option and meta read, so it counts only with request input.
|
||||
pattern = '''(?<![\w>:$\\])(?<!function )(?=(unserialize|maybe_unserialize)\s*\()(?:unserialize\s*\((?![^\n]*allowed_classes)[^\n;]*(?:\$[A-Za-z_]|php://input)|maybe_unserialize\s*\([^\n;]*(?:\$_(?:GET|POST|REQUEST|COOKIE)\b|php://input))'''
|
||||
|
||||
# --- output encoding (echo) --------------------------------------------------------------------------------
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-xss-echo-input"
|
||||
title = "PHP echo of request input without HTML escaping"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-79"
|
||||
tags = ["injection"]
|
||||
status = "shadow"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.333
|
||||
version = "1"
|
||||
pattern = '''(?m)^(?![^\n]*(?:htmlspecialchars|htmlentities|esc_html|esc_attr|esc_url|esc_js|intval|absint|json_encode|urlencode)\s*\()[^\n]*?(\becho\b|\bprint\b|<\?=)(?:[^\n;]*?[.,])?\s*\$_(?:GET|POST|REQUEST|COOKIE|SERVER)\[[^\]\n]*\]\s*(?:[.,;]|\?>)'''
|
||||
|
||||
# --- path / inclusion (filesystem, file_inclusion) ---------------------------------------------------------
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-file-inclusion-input"
|
||||
title = "PHP include/require of a path from request input"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-98"
|
||||
tags = ["filesystem_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.5
|
||||
version = "1"
|
||||
pattern = '''(?m)^(?![^\n]*\b(?:basename|realpath)\s*\()[^\n]*?(?<![\w'"$>:-])(include|include_once|require|require_once)\b(?!['"\]])\s*\(?[^\n;]*\$_(?:GET|POST|REQUEST|COOKIE)\b'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-path-input"
|
||||
title = "PHP filesystem call on a path from request input"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-22"
|
||||
tags = ["filesystem_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 1.0
|
||||
version = "1"
|
||||
pattern = '''(?m)^(?![^\n]*\b(?:basename|realpath)\s*\()[^\n]*?(?<![\w>:$\\])(fopen|file_get_contents|file_put_contents|readfile|unlink|copy|rename|scandir|opendir|mkdir|rmdir|move_uploaded_file)\s*\([^\n;]*\$_(?:GET|POST|REQUEST|COOKIE)\b'''
|
||||
|
||||
# --- untrusted destination (outbound_request, redirect) ----------------------------------------------------
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-ssrf-input"
|
||||
title = "PHP outbound request to a URL from request input"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-918"
|
||||
tags = ["network_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 1.0
|
||||
version = "1"
|
||||
pattern = '''(?<![\w>:$\\])(curl_init|curl_setopt|fsockopen|stream_socket_client|get_headers)\s*\([^\n;]*\$_(?:GET|POST|REQUEST|COOKIE)\b'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "php-open-redirect"
|
||||
title = "PHP Location header from request input"
|
||||
languages = ["php"]
|
||||
cwe = "CWE-601"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 1.0
|
||||
version = "1"
|
||||
pattern = '''(?<![\w>:$\\])(header)\s*\(\s*["']Location:[^\n;]*(?:\$_(?:GET|POST|REQUEST|COOKIE)\b|HTTP_REFERER)'''
|
||||
@@ -64,6 +64,8 @@ def test_repository_benchmark_manifests_load_and_point_to_fixtures() -> None:
|
||||
"javascript-node-web-smoke.toml",
|
||||
"javascript-vulnerable.toml",
|
||||
"mutation-python.toml",
|
||||
"php-benign.toml",
|
||||
"php-vulnerable.toml",
|
||||
"python-vulnerable.toml",
|
||||
"python-web-smoke.toml",
|
||||
"split-sink-c.toml",
|
||||
@@ -73,8 +75,8 @@ def test_repository_benchmark_manifests_load_and_point_to_fixtures() -> None:
|
||||
}
|
||||
for manifest_path in manifest_paths:
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
assert manifest.expected
|
||||
assert manifest.language in {"c_cpp", "java_web", "javascript_node_web", "python_web"}
|
||||
assert manifest.expected or manifest.name == "php-benign", "only the repaired twin expects nothing"
|
||||
assert manifest.language in {"c_cpp", "java_web", "javascript_node_web", "php_web", "python_web"}
|
||||
assert (manifest_path.parent / manifest.source.path).resolve().exists()
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""PHP quick-mode rules (ruleset/php): loading, the fixture benchmark, and the tie to the committed measurement."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import BenchmarkRun, evaluate_benchmark, load_benchmark_manifest, resolve_benchmark_source
|
||||
from openultrasast.gate import _quick_scan
|
||||
from openultrasast.ruleset import DEFAULT_RULESET_DIR, load_ruleset
|
||||
|
||||
MEASUREMENT = Path("benchmarks/measurements/2026-09-30-php-quick-rules/measurement.json")
|
||||
SEMANTIC = tomllib.loads((DEFAULT_RULESET_DIR / "semantic" / "php.toml").read_text())
|
||||
|
||||
|
||||
def _php_rules() -> dict:
|
||||
return {rule.rule_id: rule for rule in load_ruleset(DEFAULT_RULESET_DIR) if rule.rule_id.startswith("php-")}
|
||||
|
||||
|
||||
def _benchmark(name: str, *, reported_only: bool): # type: ignore[no-untyped-def]
|
||||
path = Path("benchmarks/manifests") / f"{name}.toml"
|
||||
manifest = load_benchmark_manifest(path)
|
||||
findings = _quick_scan(resolve_benchmark_source(path, manifest))
|
||||
if reported_only: # what `ousast benchmark` scores: shadow findings are kept aside, not reported
|
||||
findings = [finding for finding in findings if finding.status != "shadow"]
|
||||
run = BenchmarkRun(benchmark_run_id="t", root=Path("/tmp/t"), manifest=manifest)
|
||||
return evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None)
|
||||
|
||||
|
||||
def test_php_rules_load_compile_and_are_php_scoped() -> None:
|
||||
rules = _php_rules()
|
||||
assert len(rules) == 13
|
||||
for rule in rules.values():
|
||||
assert rule.languages == ("php",) and rule.status in {"enabled", "shadow"}
|
||||
re.compile(rule.pattern)
|
||||
|
||||
|
||||
def test_every_php_rule_sink_is_in_the_engines_vocabulary() -> None:
|
||||
"""One vocabulary: each rule's CWE is a sink family of semantic/php.toml, and the calls it names are that family's."""
|
||||
families = {sink["cwe"] for sink in SEMANTIC["sink"]}
|
||||
calls = {call.split("->")[-1] for sink in SEMANTIC["sink"] for call in sink["calls"]}
|
||||
for rule in _php_rules().values():
|
||||
assert rule.cwe in families, rule.rule_id
|
||||
named = re.findall(r"\(([a-z_|]+)\)", rule.pattern) # the rule's captured sink alternation, if any
|
||||
for group in named:
|
||||
# sprintf builds the query php-sql-sprintf-unescaped reads; curl_init takes the URL curl_exec then fetches
|
||||
assert set(group.split("|")) <= calls | {"sprintf", "curl_init"}, (rule.rule_id, group)
|
||||
|
||||
|
||||
def test_status_and_precision_are_the_committed_measurement() -> None:
|
||||
measured = json.loads(MEASUREMENT.read_text())["rules"]
|
||||
for rule_id, rule in _php_rules().items():
|
||||
assert rule.status == measured[rule_id]["status"], rule_id
|
||||
assert rule.precision_estimate == measured[rule_id]["precision_lower_bound"], rule_id
|
||||
|
||||
|
||||
def test_php_vulnerable_fixture_benchmark() -> None:
|
||||
reported = _benchmark("php-vulnerable", reported_only=True)
|
||||
assert (reported.metrics.expected_findings_total, reported.metrics.matched_findings_total) == (14, 9)
|
||||
assert reported.metrics.false_positive_findings_total == 0
|
||||
everything = _benchmark("php-vulnerable", reported_only=False)
|
||||
assert everything.metrics.matched_findings_total == 13, "the shadow rules still fire on their fixtures"
|
||||
assert [miss.rule_id for miss in everything.misses] == [None], "the cross-line flow is invisible to a line rule"
|
||||
|
||||
|
||||
def test_php_benign_twin_records_its_false_alerts() -> None:
|
||||
result = _benchmark("php-benign", reported_only=False)
|
||||
assert sorted(fp.finding_id for fp in result.false_positives) == [
|
||||
"php-file-inclusion-input:app.php:64", # the allowlist guard sits on the include's own line
|
||||
"php-sql-unquoted-concat:app.php:87", # intval() ran on the line before the concatenation
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("line", [" # eval($_POST['code']) is a comment", " // system('rm ' . $_GET['d'])"])
|
||||
def test_php_comments_do_not_fire(tmp_path: Path, line: str) -> None:
|
||||
(tmp_path / "a.php").write_text(f"<?php\n{line}\n")
|
||||
assert _quick_scan(tmp_path) == []
|
||||
(tmp_path / "a.php").write_text("<?php\n#[Pure] function f() { system('rm ' . $_GET['d']); }\n")
|
||||
assert [f.finding_id for f in _quick_scan(tmp_path)] == ["php-command-injection:a.php:2"], "#[...] is an attribute"
|
||||
@@ -121,7 +121,11 @@ def test_alerts_engine_writes_the_alerts_task_outputs_and_marks_it_done(world: d
|
||||
out = world["results"] / "validation-46" / f"{CASE}-alerts"
|
||||
rows = [json.loads(line) for line in (out / "alerts.jsonl").read_text().splitlines()]
|
||||
vulnerable, fixed = world["pins"]
|
||||
got = [(r["pin_role"], r["pin"], r["line"], r["in_fix_range"], r["source"]) for r in rows]
|
||||
quick = {(r["pin_role"], r["rule_id"], r["line"]) for r in rows if "source" not in r}
|
||||
assert ("vulnerable", "php-sql-call-composition", 3) in quick and ("fixed", "php-sql-call-composition", 3) not in quick, (
|
||||
"PHP's quick rules run beside the engine and see the intval fix"
|
||||
)
|
||||
got = [(r["pin_role"], r["pin"], r["line"], r["in_fix_range"], r["source"]) for r in rows if "source" in r]
|
||||
assert got == [("vulnerable", vulnerable, 3, True, "engine"), ("vulnerable", vulnerable, 5, False, "engine"),
|
||||
("fixed", fixed, 3, True, "engine"), ("fixed", fixed, 5, False, "engine")] # fmt: skip
|
||||
summary = json.loads((out / "summary.json").read_text())
|
||||
|
||||
@@ -99,7 +99,7 @@ def test_alerts_attribute_functions_and_mark_the_fix_on_both_pins(tmp_path: Path
|
||||
|
||||
def test_quick_coverage_is_derived_from_the_ruleset_directory(tmp_path: Path) -> None:
|
||||
shipped = alerts.quick_languages()
|
||||
assert {"python", "javascript", "typescript", "java", "c", "groovy"} <= shipped and "php" not in shipped
|
||||
assert {"python", "javascript", "typescript", "java", "c", "groovy", "php"} <= shipped, "PHP has quick rules"
|
||||
assert "php" in alerts.engine_languages(), "the engine's semantic models cover PHP"
|
||||
rules = tmp_path / "ruleset" / "php"
|
||||
rules.mkdir(parents=True)
|
||||
@@ -113,7 +113,7 @@ def test_quick_coverage_is_derived_from_the_ruleset_directory(tmp_path: Path) ->
|
||||
def test_alerts_on_an_uncovered_language_record_coverage_none_not_a_clean_zero(tmp_path: Path) -> None:
|
||||
for side in ("v", "f"):
|
||||
(tmp_path / side).mkdir()
|
||||
(tmp_path / side / "index.php").write_text('<?php\n$id = $_GET["id"];\nmysqli_query($c, "SELECT " . $id);\n')
|
||||
(tmp_path / side / "app.rb").write_text('id = params["id"]\nDB.execute("SELECT " + id)\n')
|
||||
(tmp_path / side / "app.js").write_text("module.exports = 1;\n")
|
||||
out = tmp_path / "out"
|
||||
env = {
|
||||
@@ -125,9 +125,26 @@ def test_alerts_on_an_uncovered_language_record_coverage_none_not_a_clean_zero(t
|
||||
assert (out / "alerts.jsonl").read_text() == "" and summary["alerts"] == {"vulnerable": 0, "fixed": 0}
|
||||
assert summary["coverage"] == {
|
||||
"javascript": {"coverage": "quick", "files": {"fixed": 1, "vulnerable": 1}},
|
||||
"php": {"coverage": "none", "files": {"fixed": 1, "vulnerable": 1}},
|
||||
"ruby": {"coverage": "none", "files": {"fixed": 1, "vulnerable": 1}},
|
||||
}
|
||||
assert summary["uncovered"] == ["php"] and summary["pins"] == {"vulnerable": VULNERABLE, "fixed": FIXED}
|
||||
assert summary["uncovered"] == ["ruby"] and summary["pins"] == {"vulnerable": VULNERABLE, "fixed": FIXED}
|
||||
|
||||
|
||||
def test_php_is_quick_covered_and_alerts_on_a_php_sink(tmp_path: Path) -> None:
|
||||
for side in ("v", "f"):
|
||||
(tmp_path / side).mkdir()
|
||||
(tmp_path / side / "index.php").write_text('<?php\n$id = $_GET["id"];\nmysqli_query($c, "SELECT " . $id);\n')
|
||||
out = tmp_path / "out"
|
||||
env = {
|
||||
"OUSAST_OUTPUT_DIR": str(out), "OUSAST_WORKSPACE_DIR": str(tmp_path / "v"), "OUSAST_FIXED_DIR": str(tmp_path / "f"),
|
||||
"OUSAST_VULNERABLE_PIN": VULNERABLE, "OUSAST_FIXED_PIN": FIXED,
|
||||
} # fmt: skip
|
||||
assert alerts.main(env) == 0
|
||||
summary = json.loads((out / "summary.json").read_text())
|
||||
assert summary["coverage"] == {"php": {"coverage": "quick", "files": {"fixed": 1, "vulnerable": 1}}}
|
||||
assert summary["uncovered"] == [] and summary["alerts"] == {"vulnerable": 1, "fixed": 1}
|
||||
rows = [json.loads(line) for line in (out / "alerts.jsonl").read_text().splitlines()]
|
||||
assert {(r["rule_id"], r["line"]) for r in rows} == {("php-sql-call-composition", 3)}
|
||||
|
||||
|
||||
def test_fix_ranges_new_side_are_the_fixed_pins_lines(tmp_path: Path) -> None:
|
||||
|
||||
@@ -18,8 +18,9 @@ from openultrasast.ruleset import (
|
||||
|
||||
def test_default_ruleset_loads_as_data() -> None:
|
||||
rules = load_ruleset(DEFAULT_RULESET_DIR)
|
||||
assert len(rules) == 32
|
||||
assert all(rule.status == "enabled" for rule in rules)
|
||||
php = [rule for rule in rules if rule.languages == ("php",)]
|
||||
assert len(rules) == 32 + len(php) and len(php) == 13
|
||||
assert all(rule.status == "enabled" for rule in rules if rule not in php), "PHP rules carry their measured status"
|
||||
assert all(not hasattr(rule, "severity") for rule in rules) # severity is policy-governed, not rule-local
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user