From 1741b6bed6b1ec8edd3ea23f644084cc4bb9cf10 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:47:06 +0200 Subject: [PATCH] feat(model): phase 2 measured (weights v1) and `carried` reported by the evidence branch budget_at_recall by a real evidence pass over every region: pmpro 391 -> 116, wpstatistics 504 -> 223, mwwpform 513 -> 292, vampi 16 -> 16. The gate (< 50) is not met. pmpro's CVE is inside its 121 tier-4 regions; WP Statistics's and MW WP Form's are one tier low because `carried` -- stage one of the two-stage join -- was not reported as evidence. taint.sc now reports it where a sink exists, and the vector reads the query's answer when given. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .kiro/specs/flow-aware-ranking/brief.md | 19 +++ .../baselines/2026-09-11-evidence-rank.json | 140 ++++++++++++++++++ src/openultrasast/cpg/queries/taint.sc | 8 + src/openultrasast/model/evidence.py | 4 +- tests/test_model_evidence.py | 22 +++ 5 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 benchmarks/ranking/baselines/2026-09-11-evidence-rank.json diff --git a/.kiro/specs/flow-aware-ranking/brief.md b/.kiro/specs/flow-aware-ranking/brief.md index 18c9a78..151eb46 100644 --- a/.kiro/specs/flow-aware-ranking/brief.md +++ b/.kiro/specs/flow-aware-ranking/brief.md @@ -137,6 +137,25 @@ A phase that cannot show that is not an improvement, it is a story. after every region with evidence -- the scan does not know it is safe, it knows nothing -- which is a known cost of this phase for families like `access_control` and is noted here rather than hidden. + **Measured 2026-09-11, weights v1** (`benchmarks/ranking/baselines/2026-09-11-evidence-rank.json`), the + evidence pass over EVERY region of each repository, Joern 4.0.623: + + | repository | pairs | evidence pass | tiers 0 / 1 / 2 / 3 / 4 | `budget_at_recall` static → evidence | + |---|---|---|---|---| + | pmpro | 22,315 | 255 s | 21,275 / 74 / 86 / 759 / 121 | **391 → 116** | + | wpstatistics | 8,359 | 156 s | 7,468 / 77 / 5 / 726 / 83 | **504 → 223** | + | mwwpform | 4,915 | 68 s | 4,511 / 37 / 0 / 357 / 10 | **513 → 292** | + | vampi | 125 | 16 s | 123 / 1 / 0 / 0 / 1 | 16 → 16 | + + Concentration of 2.2× to 3.4× on the three PHP repositories, from exact facts and a seven-entry weight + table; the gate (< 50 everywhere) is not met. Two readings, one per shape of miss: pmpro's CVE is + INSIDE its 121 tier-4 regions, so what remains there is order within the top tier -- the tie-heavy part + this brief handed to phase 3. WP Statistics's `record()` and MW WP Form's `_delete_files()` are in + tier 3, one below where their sources say they belong, because `carried` -- stage one of the two-stage + join, which the arbiter computes anyway -- was not yet reported by the evidence branch. It is now + (`taint.sc` reports it only where a sink exists, so tier-0 pairs pay nothing); the re-measurement is + in flight and is the number phase 2 stands or falls on. + - **Phase 3 — the LLM ranker.** Same evidence, ordered by a model. Gate: beats phase 2 leave-one-repository-out. If it does not, phase 2 ships and phase 3 does not. - **Phase 4 — break the closed loop.** An exploration slice (~10% of budget sampled outside the top-K) and diff --git a/benchmarks/ranking/baselines/2026-09-11-evidence-rank.json b/benchmarks/ranking/baselines/2026-09-11-evidence-rank.json new file mode 100644 index 0000000..c843902 --- /dev/null +++ b/benchmarks/ranking/baselines/2026-09-11-evidence-rank.json @@ -0,0 +1,140 @@ +{ + "note": "phase 2, weights v1: order by a real evidence pass over every region (position.py --evidence), Joern 4.0.623", + "results": [ + { + "repo": "mwwpform", + "regions": 983, + "distinct_ranks": 5, + "largest_tier_share": 0.817, + "largest_tier_rank": 0.3, + "budget_at_recall": 292, + "known": [ + { + "id": "CVE-2023-6559", + "family": "path", + "site": "classes/services/class.mail.php:259:_delete_files", + "position": 291, + "rank": 0.3, + "in_scope": true, + "reached_transitively": false + } + ], + "build_seconds": 47.6, + "evidence_seconds": 67.5, + "pairs": 4915, + "pairs_unanswered": 0, + "tier_counts": { + "0": 4511, + "1": 37, + "3": 357, + "4": 10 + } + }, + { + "repo": "pmpro", + "regions": 4463, + "distinct_ranks": 5, + "largest_tier_share": 0.838, + "largest_tier_rank": 0.3, + "budget_at_recall": 116, + "known": [ + { + "id": "CVE-2023-23488", + "family": "injection", + "site": "classes/class.memberorder.php:936:getMemberOrderByCode", + "position": 115, + "rank": 0.3, + "in_scope": true, + "reached_transitively": false + } + ], + "build_seconds": 84.8, + "evidence_seconds": 255.1, + "pairs": 22315, + "pairs_unanswered": 0, + "tier_counts": { + "0": 21275, + "1": 74, + "2": 86, + "3": 759, + "4": 121 + } + }, + { + "repo": "vampi", + "regions": 25, + "distinct_ranks": 5, + "largest_tier_share": 0.36, + "largest_tier_rank": 1.0, + "budget_at_recall": 16, + "known": [ + { + "id": "VAmPI-SQLI", + "family": "injection", + "site": "models/user_model.py:73:get_user", + "position": null, + "rank": null, + "in_scope": true, + "reached_transitively": true + }, + { + "id": "VAmPI-BOLA", + "family": "access_control", + "site": "api_views/users.py:187:update_password", + "position": 15, + "rank": 0.8, + "in_scope": true, + "reached_transitively": false + }, + { + "id": "VAmPI-BOLA-BOOKS", + "family": "access_control", + "site": "api_views/books.py:51:get_by_title", + "position": 10, + "rank": 0.8, + "in_scope": true, + "reached_transitively": false + } + ], + "build_seconds": 5.2, + "evidence_seconds": 15.8, + "pairs": 125, + "pairs_unanswered": 0, + "tier_counts": { + "0": 123, + "1": 1, + "4": 1 + } + }, + { + "repo": "wpstatistics", + "regions": 1673, + "distinct_ranks": 6, + "largest_tier_share": 0.753, + "largest_tier_rank": 0.3, + "budget_at_recall": 223, + "known": [ + { + "id": "CVE-2022-25148", + "family": "injection", + "site": "includes/class-wp-statistics-pages.php:225:record", + "position": 222, + "rank": 0.3, + "in_scope": true, + "reached_transitively": false + } + ], + "build_seconds": 106.4, + "evidence_seconds": 156.2, + "pairs": 8359, + "pairs_unanswered": 0, + "tier_counts": { + "0": 7468, + "1": 77, + "2": 5, + "3": 726, + "4": 83 + } + } + ] +} diff --git a/src/openultrasast/cpg/queries/taint.sc b/src/openultrasast/cpg/queries/taint.sc index 3571d6f..f135da1 100644 --- a/src/openultrasast/cpg/queries/taint.sc +++ b/src/openultrasast/cpg/queries/taint.sc @@ -554,11 +554,19 @@ // the same answer for every request of one family and a repository-wide walk per request turned a // milliseconds-per-pair query into minutes. val familyInRepo = familyInRepoMemo.getOrElseUpdate(sinksS, cpg.call.exists(c => sinkNames.exists(n => sinkMatches(c, n)))) + // `carried`: stage one of the two-stage join, reused as evidence (flow-aware-ranking, phase 2). A field + // read in scope whose assignment is fed by a source, or an array key a registered hook callback fills + // from one -- computed and memoised for the arbiter anyway, and the exact fact that puts + // `_delete_files()` (fed by `$this->attachments`) and `record()` (fed by a filter) where their sources + // say, instead of one tier below every region with a `$_GET` in its own body. Asked only where there + // is a sink to carry to, so the 21,000 tier-0 pairs of a plugin pay nothing for it. + val carried = sinkList.nonEmpty && familyInRepo && (fieldList.nonEmpty || hookList.nonEmpty) val summary = ujson.Obj( "kind" -> "summary", "sinks" -> sinkList.size, "sourceLocal" -> sourceLocal, "sourceNear" -> sourceNear, + "carried" -> carried, "familyInRepo" -> familyInRepo ) val perSink = sinkList.map { sink => diff --git a/src/openultrasast/model/evidence.py b/src/openultrasast/model/evidence.py index 1509a4f..d6ac80b 100644 --- a/src/openultrasast/model/evidence.py +++ b/src/openultrasast/model/evidence.py @@ -156,7 +156,9 @@ def evidence_from_rows( source_near=_as_bool(summary.get("sourceNear")), entry=entry, access_declared_public=access_declared_public, - carried=carried, + # The query's own answer when it gave one (phase 2: stage one of the two-stage join, reused); + # the caller's otherwise; None where neither knows, which is not False. + carried=_as_bool(summary["carried"]) if "carried" in summary else carried, bound_names=tuple(bound_names), ) diff --git a/tests/test_model_evidence.py b/tests/test_model_evidence.py index 133eb39..6755efc 100644 --- a/tests/test_model_evidence.py +++ b/tests/test_model_evidence.py @@ -123,3 +123,25 @@ def test_the_score_orders_within_a_tier_and_never_across_one() -> None: cleansed = Evidence(sinks=(sink(1), sink(2, cleansed=True)), source_near=True) assert cleansed.score < one.score, "a cleansed sink beside an open one counts against the pair" assert sorted([many, one, local], key=lambda e: e.order_key, reverse=True) == [local, many, one] + + +def test_the_query_may_answer_carried_itself() -> None: + """Stage one of the two-stage join, reported by the query as `carried`. When present it is the answer; + when absent the caller's value stands, and absent-and-unknown stays None rather than becoming False.""" + from openultrasast.model.evidence import TIER_OPEN, TIER_OPEN_PUBLIC, evidence_from_rows + + sink = { + "kind": "sink", + "sink": "unlink($f)", + "sinkLine": "259", + "sinkMethod": "_delete_files", + "sinkArity": 1, + "sinkArg0Literal": False, + "cleansedOnCall": False, + } + said = evidence_from_rows([{"kind": "summary", "sinks": 1, "sourceNear": True, "carried": True, "familyInRepo": True}, sink]) + assert said is not None and said.carried is True and said.tier == TIER_OPEN_PUBLIC + silent = evidence_from_rows([{"kind": "summary", "sinks": 1, "sourceNear": True, "familyInRepo": True}, sink]) + assert silent is not None and silent.carried is None and silent.tier == TIER_OPEN + told = evidence_from_rows([{"kind": "summary", "sinks": 1, "sourceNear": True, "familyInRepo": True}, sink], carried=True) + assert told is not None and told.carried is True