mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(model): phase 2 measured (weights v1) and carried reported by the evidence branch
budget_at_recall by a real evidence pass over every region: pmpro 391 -> 116, wpstatistics 504 -> 223, mwwpform 513 -> 292, vampi 16 -> 16. The gate (< 50) is not met. pmpro's CVE is inside its 121 tier-4 regions; WP Statistics's and MW WP Form's are one tier low because `carried` -- stage one of the two-stage join -- was not reported as evidence. taint.sc now reports it where a sink exists, and the vector reads the query's answer when given. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
9f881b433e
commit
1741b6bed6
@@ -137,6 +137,25 @@ A phase that cannot show that is not an improvement, it is a story.
|
||||
after every region with evidence -- the scan does not know it is safe, it knows nothing -- which is a
|
||||
known cost of this phase for families like `access_control` and is noted here rather than hidden.
|
||||
|
||||
**Measured 2026-09-11, weights v1** (`benchmarks/ranking/baselines/2026-09-11-evidence-rank.json`), the
|
||||
evidence pass over EVERY region of each repository, Joern 4.0.623:
|
||||
|
||||
| repository | pairs | evidence pass | tiers 0 / 1 / 2 / 3 / 4 | `budget_at_recall` static → evidence |
|
||||
|---|---|---|---|---|
|
||||
| pmpro | 22,315 | 255 s | 21,275 / 74 / 86 / 759 / 121 | **391 → 116** |
|
||||
| wpstatistics | 8,359 | 156 s | 7,468 / 77 / 5 / 726 / 83 | **504 → 223** |
|
||||
| mwwpform | 4,915 | 68 s | 4,511 / 37 / 0 / 357 / 10 | **513 → 292** |
|
||||
| vampi | 125 | 16 s | 123 / 1 / 0 / 0 / 1 | 16 → 16 |
|
||||
|
||||
Concentration of 2.2× to 3.4× on the three PHP repositories, from exact facts and a seven-entry weight
|
||||
table; the gate (< 50 everywhere) is not met. Two readings, one per shape of miss: pmpro's CVE is
|
||||
INSIDE its 121 tier-4 regions, so what remains there is order within the top tier -- the tie-heavy part
|
||||
this brief handed to phase 3. WP Statistics's `record()` and MW WP Form's `_delete_files()` are in
|
||||
tier 3, one below where their sources say they belong, because `carried` -- stage one of the two-stage
|
||||
join, which the arbiter computes anyway -- was not yet reported by the evidence branch. It is now
|
||||
(`taint.sc` reports it only where a sink exists, so tier-0 pairs pay nothing); the re-measurement is
|
||||
in flight and is the number phase 2 stands or falls on.
|
||||
|
||||
- **Phase 3 — the LLM ranker.** Same evidence, ordered by a model. Gate: beats phase 2 leave-one-repository-out.
|
||||
If it does not, phase 2 ships and phase 3 does not.
|
||||
- **Phase 4 — break the closed loop.** An exploration slice (~10% of budget sampled outside the top-K) and
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"note": "phase 2, weights v1: order by a real evidence pass over every region (position.py --evidence), Joern 4.0.623",
|
||||
"results": [
|
||||
{
|
||||
"repo": "mwwpform",
|
||||
"regions": 983,
|
||||
"distinct_ranks": 5,
|
||||
"largest_tier_share": 0.817,
|
||||
"largest_tier_rank": 0.3,
|
||||
"budget_at_recall": 292,
|
||||
"known": [
|
||||
{
|
||||
"id": "CVE-2023-6559",
|
||||
"family": "path",
|
||||
"site": "classes/services/class.mail.php:259:_delete_files",
|
||||
"position": 291,
|
||||
"rank": 0.3,
|
||||
"in_scope": true,
|
||||
"reached_transitively": false
|
||||
}
|
||||
],
|
||||
"build_seconds": 47.6,
|
||||
"evidence_seconds": 67.5,
|
||||
"pairs": 4915,
|
||||
"pairs_unanswered": 0,
|
||||
"tier_counts": {
|
||||
"0": 4511,
|
||||
"1": 37,
|
||||
"3": 357,
|
||||
"4": 10
|
||||
}
|
||||
},
|
||||
{
|
||||
"repo": "pmpro",
|
||||
"regions": 4463,
|
||||
"distinct_ranks": 5,
|
||||
"largest_tier_share": 0.838,
|
||||
"largest_tier_rank": 0.3,
|
||||
"budget_at_recall": 116,
|
||||
"known": [
|
||||
{
|
||||
"id": "CVE-2023-23488",
|
||||
"family": "injection",
|
||||
"site": "classes/class.memberorder.php:936:getMemberOrderByCode",
|
||||
"position": 115,
|
||||
"rank": 0.3,
|
||||
"in_scope": true,
|
||||
"reached_transitively": false
|
||||
}
|
||||
],
|
||||
"build_seconds": 84.8,
|
||||
"evidence_seconds": 255.1,
|
||||
"pairs": 22315,
|
||||
"pairs_unanswered": 0,
|
||||
"tier_counts": {
|
||||
"0": 21275,
|
||||
"1": 74,
|
||||
"2": 86,
|
||||
"3": 759,
|
||||
"4": 121
|
||||
}
|
||||
},
|
||||
{
|
||||
"repo": "vampi",
|
||||
"regions": 25,
|
||||
"distinct_ranks": 5,
|
||||
"largest_tier_share": 0.36,
|
||||
"largest_tier_rank": 1.0,
|
||||
"budget_at_recall": 16,
|
||||
"known": [
|
||||
{
|
||||
"id": "VAmPI-SQLI",
|
||||
"family": "injection",
|
||||
"site": "models/user_model.py:73:get_user",
|
||||
"position": null,
|
||||
"rank": null,
|
||||
"in_scope": true,
|
||||
"reached_transitively": true
|
||||
},
|
||||
{
|
||||
"id": "VAmPI-BOLA",
|
||||
"family": "access_control",
|
||||
"site": "api_views/users.py:187:update_password",
|
||||
"position": 15,
|
||||
"rank": 0.8,
|
||||
"in_scope": true,
|
||||
"reached_transitively": false
|
||||
},
|
||||
{
|
||||
"id": "VAmPI-BOLA-BOOKS",
|
||||
"family": "access_control",
|
||||
"site": "api_views/books.py:51:get_by_title",
|
||||
"position": 10,
|
||||
"rank": 0.8,
|
||||
"in_scope": true,
|
||||
"reached_transitively": false
|
||||
}
|
||||
],
|
||||
"build_seconds": 5.2,
|
||||
"evidence_seconds": 15.8,
|
||||
"pairs": 125,
|
||||
"pairs_unanswered": 0,
|
||||
"tier_counts": {
|
||||
"0": 123,
|
||||
"1": 1,
|
||||
"4": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"repo": "wpstatistics",
|
||||
"regions": 1673,
|
||||
"distinct_ranks": 6,
|
||||
"largest_tier_share": 0.753,
|
||||
"largest_tier_rank": 0.3,
|
||||
"budget_at_recall": 223,
|
||||
"known": [
|
||||
{
|
||||
"id": "CVE-2022-25148",
|
||||
"family": "injection",
|
||||
"site": "includes/class-wp-statistics-pages.php:225:record",
|
||||
"position": 222,
|
||||
"rank": 0.3,
|
||||
"in_scope": true,
|
||||
"reached_transitively": false
|
||||
}
|
||||
],
|
||||
"build_seconds": 106.4,
|
||||
"evidence_seconds": 156.2,
|
||||
"pairs": 8359,
|
||||
"pairs_unanswered": 0,
|
||||
"tier_counts": {
|
||||
"0": 7468,
|
||||
"1": 77,
|
||||
"2": 5,
|
||||
"3": 726,
|
||||
"4": 83
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -554,11 +554,19 @@
|
||||
// the same answer for every request of one family and a repository-wide walk per request turned a
|
||||
// milliseconds-per-pair query into minutes.
|
||||
val familyInRepo = familyInRepoMemo.getOrElseUpdate(sinksS, cpg.call.exists(c => sinkNames.exists(n => sinkMatches(c, n))))
|
||||
// `carried`: stage one of the two-stage join, reused as evidence (flow-aware-ranking, phase 2). A field
|
||||
// read in scope whose assignment is fed by a source, or an array key a registered hook callback fills
|
||||
// from one -- computed and memoised for the arbiter anyway, and the exact fact that puts
|
||||
// `_delete_files()` (fed by `$this->attachments`) and `record()` (fed by a filter) where their sources
|
||||
// say, instead of one tier below every region with a `$_GET` in its own body. Asked only where there
|
||||
// is a sink to carry to, so the 21,000 tier-0 pairs of a plugin pay nothing for it.
|
||||
val carried = sinkList.nonEmpty && familyInRepo && (fieldList.nonEmpty || hookList.nonEmpty)
|
||||
val summary = ujson.Obj(
|
||||
"kind" -> "summary",
|
||||
"sinks" -> sinkList.size,
|
||||
"sourceLocal" -> sourceLocal,
|
||||
"sourceNear" -> sourceNear,
|
||||
"carried" -> carried,
|
||||
"familyInRepo" -> familyInRepo
|
||||
)
|
||||
val perSink = sinkList.map { sink =>
|
||||
|
||||
@@ -156,7 +156,9 @@ def evidence_from_rows(
|
||||
source_near=_as_bool(summary.get("sourceNear")),
|
||||
entry=entry,
|
||||
access_declared_public=access_declared_public,
|
||||
carried=carried,
|
||||
# The query's own answer when it gave one (phase 2: stage one of the two-stage join, reused);
|
||||
# the caller's otherwise; None where neither knows, which is not False.
|
||||
carried=_as_bool(summary["carried"]) if "carried" in summary else carried,
|
||||
bound_names=tuple(bound_names),
|
||||
)
|
||||
|
||||
|
||||
@@ -123,3 +123,25 @@ def test_the_score_orders_within_a_tier_and_never_across_one() -> None:
|
||||
cleansed = Evidence(sinks=(sink(1), sink(2, cleansed=True)), source_near=True)
|
||||
assert cleansed.score < one.score, "a cleansed sink beside an open one counts against the pair"
|
||||
assert sorted([many, one, local], key=lambda e: e.order_key, reverse=True) == [local, many, one]
|
||||
|
||||
|
||||
def test_the_query_may_answer_carried_itself() -> None:
|
||||
"""Stage one of the two-stage join, reported by the query as `carried`. When present it is the answer;
|
||||
when absent the caller's value stands, and absent-and-unknown stays None rather than becoming False."""
|
||||
from openultrasast.model.evidence import TIER_OPEN, TIER_OPEN_PUBLIC, evidence_from_rows
|
||||
|
||||
sink = {
|
||||
"kind": "sink",
|
||||
"sink": "unlink($f)",
|
||||
"sinkLine": "259",
|
||||
"sinkMethod": "_delete_files",
|
||||
"sinkArity": 1,
|
||||
"sinkArg0Literal": False,
|
||||
"cleansedOnCall": False,
|
||||
}
|
||||
said = evidence_from_rows([{"kind": "summary", "sinks": 1, "sourceNear": True, "carried": True, "familyInRepo": True}, sink])
|
||||
assert said is not None and said.carried is True and said.tier == TIER_OPEN_PUBLIC
|
||||
silent = evidence_from_rows([{"kind": "summary", "sinks": 1, "sourceNear": True, "familyInRepo": True}, sink])
|
||||
assert silent is not None and silent.carried is None and silent.tier == TIER_OPEN
|
||||
told = evidence_from_rows([{"kind": "summary", "sinks": 1, "sourceNear": True, "familyInRepo": True}, sink], carried=True)
|
||||
assert told is not None and told.carried is True
|
||||
|
||||
Reference in New Issue
Block a user