mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
flow-aware-ranking phase 0: measure before changing anything.
THE BASELINE, committed as benchmarks/ranking/baselines/2026-09-10-access-rank.json
from benchmarks/ranking/position.py, which sorts regions exactly as the scan does:
pmpro 4,463 regions 83.8% at 0.30 CVE at 390 budget_at_recall 391
mwwpform 983 regions 81.7% at 0.30 CVE at 512 budget_at_recall 513
wpstatistics 1,673 regions 75.3% at 0.30 CVE at 503 budget_at_recall 504
vampi 25 regions BOLA at 15, 10; SQLI reached transitively
That corrects the brief's own cross-assessment: "concentration, not inclusion"
was measured on pmpro alone. On the other two plugins the CVE sits OUTSIDE a
500-region budget and is never examined; the slices found those CVEs only
because a slice has too few regions to exclude anything.
THE INSTRUMENT: `evidenceOnly` mode on the taint query -- what is in scope,
which sink calls, what shape, whether a sanitizer wraps THEIR argument, whether
a source is local or reachable -- returning before reachableByFlows is ever
called. model/evidence.py turns its rows into a vector per (region, family)
and a tier; scan.py records every pair's tier on the result and acts on none of
it. Every later phase is measured against a record made by scans that did not
know they were being measured.
2,500 pairs in 98.9s incl. JVM load, ~40 ms per pair
tier 0 -- no sink of that family in reach: 2,161 of 2,500 = 86%
Two earlier attempts ran past an hour each with no payload, and both were my
fault: repository-wide walks repeated per request -- the sink scan, the
reachable-method BFS, familyInRepo. Memoised across the batch.
AND THE SAME FIX REACHES THE DATAFLOW MODE. sinkCalls and reachableMethods do
not depend on the family, and were recomputed for all 2,500 requests. The
identical 250-request set with both joins on:
2,026.3s -> 287.0s 8,105 -> 1,148 ms per request same 5,779 rows
Most of what 5.13 measured as "the base taint query" was never dataflow. A 7x
speedup with nothing about the analysis changed; both corrections recorded in
the specs. Combined with 86% tier 0, the whole-repository taint query that
timed out at 40 minutes is ~339 real requests at ~1.1s: about six minutes.
Also: the module audit manifest is regenerated (91 load-bearing, 7 standalone,
0 orphaned) because evidence.py is now imported by the scan, and the
failed-query test is order-independent since the evidence pass records its own
failure before taint's.
869 tests pass; query compile test 4 passed; gates byte-identical.
Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
100 lines
4.3 KiB
Python
100 lines
4.3 KiB
Python
"""The evidence vector and its tiers, on canned rows (flow-aware-ranking phase 0)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from openultrasast.model.evidence import (
|
|
TIER_CLEANSED,
|
|
TIER_EXCLUDE,
|
|
TIER_NO_SOURCE,
|
|
TIER_OPEN,
|
|
TIER_OPEN_PUBLIC,
|
|
evidence_from_rows,
|
|
)
|
|
|
|
SUMMARY = {"kind": "summary", "sinks": 1, "sourceLocal": False, "sourceNear": False, "familyInRepo": True}
|
|
|
|
|
|
def _sink(code: str, *, literal: bool = False, cleansed: bool = False, arity: int = 1) -> dict[str, object]:
|
|
return {
|
|
"kind": "sink",
|
|
"sink": code,
|
|
"sinkLine": "936",
|
|
"sinkMethod": "f",
|
|
"sinkArity": arity,
|
|
"sinkArg0Literal": literal,
|
|
"cleansedOnCall": cleansed,
|
|
}
|
|
|
|
|
|
def test_a_failed_query_is_not_a_tier() -> None:
|
|
"""No rows at all means the query did not answer. That is a degradation, never tier 0."""
|
|
assert evidence_from_rows(None) is None
|
|
assert evidence_from_rows([]) is None
|
|
|
|
|
|
def test_no_sink_in_scope_is_tier_zero_and_exact() -> None:
|
|
"""The only tier that excludes. A family with no sink in reach cannot produce a flow, in any language."""
|
|
e = evidence_from_rows([{**SUMMARY, "sinks": 0}], entry=True)
|
|
assert e is not None and e.tier == TIER_EXCLUDE
|
|
|
|
|
|
def test_no_sink_anywhere_in_the_repository_is_tier_zero_at_any_depth() -> None:
|
|
"""PMPro never calls `unserialize`: all 487 deserialization requests were asked of nothing."""
|
|
e = evidence_from_rows([{**SUMMARY, "familyInRepo": False}, _sink("x($y)")], entry=True)
|
|
assert e is not None and e.tier == TIER_EXCLUDE
|
|
|
|
|
|
def test_a_sink_with_no_source_is_asked_last() -> None:
|
|
e = evidence_from_rows([SUMMARY, _sink('$wpdb->get_row("... $id")')])
|
|
assert e is not None and e.tier == TIER_NO_SOURCE
|
|
|
|
|
|
def test_the_cve_shape_is_the_top_tier() -> None:
|
|
"""CVE-2023-23488: an entry-point method whose sink concatenates a parameter into SQL, with `esc_sql`
|
|
present in the FUNCTION but wrapping a different statement -- so `cleansedOnCall` is false here."""
|
|
e = evidence_from_rows(
|
|
[SUMMARY, _sink('$wpdb->get_var("... WHERE code = \'" . $code . "\'")', cleansed=False)],
|
|
entry=True,
|
|
)
|
|
assert e is not None and e.tier == TIER_OPEN
|
|
assert len(e.open_sinks) == 1
|
|
|
|
|
|
def test_cleansing_on_the_call_itself_lowers_the_tier() -> None:
|
|
"""The fixed side of that pair wraps the value in `$wpdb->prepare`, which is an ancestor of the argument."""
|
|
e = evidence_from_rows([SUMMARY, _sink('$wpdb->get_var($wpdb->prepare("... %s", $code))', cleansed=True)], entry=True)
|
|
assert e is not None and e.tier == TIER_CLEANSED
|
|
|
|
|
|
def test_a_literal_argument_cannot_carry_taint() -> None:
|
|
e = evidence_from_rows([SUMMARY, _sink('system("ls")', literal=True)], entry=True)
|
|
assert e is not None and e.tier == TIER_CLEANSED
|
|
|
|
|
|
def test_a_bound_shape_is_the_arbiters_own_rule() -> None:
|
|
"""`execute(sql, params)` binds where `execute(sql + x)` interpolates; same rule the taint verdict uses."""
|
|
e = evidence_from_rows([SUMMARY, _sink("cursor.execute(sql, params)", arity=2)], entry=True, bound_names=("execute",))
|
|
assert e is not None and e.tier == TIER_CLEANSED
|
|
|
|
|
|
def test_declared_public_or_a_local_source_asks_very_first() -> None:
|
|
public = evidence_from_rows([SUMMARY, _sink("unlink($f)")], entry=True, access_declared_public=True)
|
|
local = evidence_from_rows([{**SUMMARY, "sourceLocal": True}, _sink("unlink($_GET['f'])")])
|
|
assert public is not None and public.tier == TIER_OPEN_PUBLIC
|
|
assert local is not None and local.tier == TIER_OPEN_PUBLIC
|
|
|
|
|
|
def test_a_carried_field_counts_as_a_source() -> None:
|
|
"""`_delete_files()` has no parameters; its taint arrives as `$this->attachments`. Stage one of the join
|
|
already knows that, and the vector must not demote the region for lacking a direct source."""
|
|
e = evidence_from_rows([SUMMARY, _sink("unlink($file)")], carried=True)
|
|
assert e is not None and e.tier == TIER_OPEN_PUBLIC
|
|
|
|
|
|
def test_a_stringified_boolean_is_read_as_its_value_not_its_truthiness() -> None:
|
|
"""`bool("False")` is True. A serialiser that chose strings would make every open sink read as cleansed."""
|
|
rows = [{**SUMMARY, "sourceLocal": "False", "familyInRepo": "true"}, _sink("unlink($f)")]
|
|
rows[1]["cleansedOnCall"] = "False"
|
|
e = evidence_from_rows(rows, entry=True)
|
|
assert e is not None and e.tier == TIER_OPEN, "a 'False' string must not turn an open sink into a cleansed one"
|