From 9a8e9f083f51fd8f434373af6c65da501081a9b2 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:18:20 +0200 Subject: [PATCH] feat(ranking): phase 1 -- tier-0 taint requests are no longer asked A pair with no sink of its family in reach cannot yield a flow, in any language under any dataflow model, so its taint request is never issued. The count is recorded as requests_pruned -- exact, so not a degradation -- and a test asserts that every other pair's request still goes out. 637-file plugin, 500-region budget tier_counts {0: 2161, 1: 60, 2: 20, 3: 211, 4: 48} pruned 2,161 of 2,500 = 86% evidence pass 94.5s taint 339 requests -> 2,400s TIMEOUT Gate met and exceeded; the scan still does not finish, and the reason corrects phase 0. "1,148 ms per request" was an average over a 250-request set that was 88% tier 0 and answered instantly; the ~30 real requests in it cost ~9.5s each. Pruning removes exactly the cheap questions and keeps the expensive ones, so it cannot make them faster. The honest cost of a taint question that CAN have an answer is 7-10s at callDepth=3, and that is what phase 2 and 5.13 design against -- not 1.1s, and not 6.6s either. Both specs corrected. 869 tests pass. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .kiro/specs/contributor-scan/tasks.md | 6 ++- .kiro/specs/flow-aware-ranking/brief.md | 11 +++++ src/openultrasast/model/scan.py | 23 ++++++++++- tests/test_model_scan.py | 53 +++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 3 deletions(-) diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index 612c38c..fd1d5bf 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -826,8 +826,10 @@ independent of the region's family, both repeated 2,500 times. Memoised across the batch (found while building flow-aware-ranking's evidence mode, which asks no dataflow and still took an hour), the same 250 requests with both joins on: **2,026.3 s → 287.0 s, 8,105 → 1,148 ms per request, identical 5,779 - rows.** A 7× speedup with nothing about the analysis changed. The base cost of a taint request is about - 1.1 s, and the levers above still apply to it. + rows.** A 7× speedup with nothing about the analysis changed -- but "1.1 s" is an average over a set + that was 88% tier-0 pairs answering instantly. With those pruned (flow-aware-ranking phase 1), the 339 + requests that CAN have an answer still exceed 2,400 s: **7–10 s per real request at `callDepth = 3`**. + The memoisation was real and stays; the per-request figure to design against is that one. - **The estimate this replaces was wrong and worth recording as wrong.** ">0.94s per request" came from assuming taint COMPLETED within its 2,400s ceiling. It timed out, so that was a floor presented as a figure. The measured cost is seven times higher. diff --git a/.kiro/specs/flow-aware-ranking/brief.md b/.kiro/specs/flow-aware-ranking/brief.md index 1c9f7b6..e812614 100644 --- a/.kiro/specs/flow-aware-ranking/brief.md +++ b/.kiro/specs/flow-aware-ranking/brief.md @@ -93,6 +93,17 @@ A phase that cannot show that is not an improvement, it is a story. - **Phase 1 — tier 0 only, no model.** Exact pruning. Gate: **53% fewer taint requests issued, measured** — and nothing more. 1,177 requests at 6.63 s is still 2.2 hours, so this phase does NOT make the scan finish; it halves the problem. Claiming completion here would send someone looking for a bug where there is none. + + **Done 2026-09-10, gate exceeded, scan still does not finish — and the reason corrects phase 0's cost + figure.** On the 637-file plugin: `tier_counts = {0: 2161, 1: 60, 2: 20, 3: 211, 4: 48}`, **2,161 of 2,500 + requests pruned (86%)**, evidence pass 94.5 s. The remaining **339 requests still hit the 2,400 s ceiling** + — more than 7 s each. + + Phase 0 reported 1,148 ms per request after memoisation. That was an average over a 250-request set of + which 88% were tier 0 and returned instantly; the ~30 real requests in it cost about 9.5 s each. Pruning + removes exactly the cheap questions and keeps the expensive ones, so it cannot make those faster. The + honest per-request cost of a taint question that *can* have an answer is **7–10 s at `callDepth = 3`**, + and that is the number phase 2 and 5.13 must work against — not 1.1 s, and not 6.6 s either. - **Phase 2 — heuristic tiers, still no model.** The vector and tiers defined below, ordered within a tier by a published weighted sum. Gate: `budget_at_recall` — the smallest K holding every pinned CVE — **below 50 on every pinned repository**, and `pmpro`'s taint query completing at that K. This is the phase that diff --git a/src/openultrasast/model/scan.py b/src/openultrasast/model/scan.py index 37f65c8..1051cde 100644 --- a/src/openultrasast/model/scan.py +++ b/src/openultrasast/model/scan.py @@ -26,7 +26,7 @@ from pathlib import Path from typing import Any from ..cpg.backend import CpgResult -from .evidence import evidence_from_rows +from .evidence import TIER_EXCLUDE, evidence_from_rows from .candidates import enumerate_candidates from .config_value import request_params as config_params from .dominance import request_params as dominance_params @@ -70,6 +70,11 @@ class ScanBudget: # what is asked or in what order. One extra query invocation without dataflow -- a JVM load, then # milliseconds per pair. tiering: bool = True + # Phase 1 of flow-aware-ranking: do not ask a taint question whose answer is known in advance. A pair at + # tier 0 has no sink of its family in reach, and a family with no sink cannot produce a flow -- in any + # language, under any dataflow model. Skipping it cannot lose a finding. It is the only tier that + # excludes, which is why it is the only one allowed to act before the ranker is measured. + prune_tier0: bool = True @dataclass(frozen=True) @@ -97,6 +102,9 @@ class ModelScanResult: # Recorded, not acted on: this is what "the ranker got better" will be measured against. tiers: tuple[tuple[str, str, str, int], ...] = () tier_counts: Mapping[int, int] = field(default_factory=dict) + # Taint requests not issued because their pair was tier 0. Exact, so not a degradation: nothing that + # could have been found was skipped. Reported so the request count is legible. + requests_pruned: int = 0 @property def arbitrate_seconds(self) -> float: @@ -190,6 +198,7 @@ def scan_repository( # no `reachableByFlows`. Recorded on the result and NOT used to reorder or skip anything -- that is a # later phase, and it must be measured against this record before it is allowed to change behaviour. tiers: list[tuple[str, str, str, int]] = [] + tier_by_rid: dict[str, int] = {} per_kind: dict[str, float] = {} if limits.tiering and callable(getattr(cpg, "run_batch", None)): grouped_taint = _grouped(work, hook_callbacks=hooks).get("taint", {}) @@ -219,6 +228,7 @@ def scan_repository( ) if evidence is not None: tiers.append((region.path, region.function or "", getattr(spec, "family", ""), evidence.tier)) + tier_by_rid[rid] = evidence.tier # Phase 2: ONE invocation per query kind. JVM startup dominates a repository scan -- a call per region # per family put a ten-line file at four minutes and a thousand regions at roughly fifty hours -- while @@ -226,7 +236,17 @@ def scan_repository( rows_by_id: dict[str, list[object]] = {} census_reported = False query_started = time.monotonic() + pruned = 0 for kind, requests in _grouped(work, hook_callbacks=hooks).items(): + if kind == "taint" and limits.prune_tier0 and tier_by_rid: + # Tier 0 is exact: a family with no sink in reach cannot yield a flow. Measured on a 637-file + # plugin, that is 2,161 of 2,500 requests -- 86% of a query that timed out at forty minutes. + keep = {rid: params for rid, params in requests.items() if tier_by_rid.get(rid, TIER_EXCLUDE + 1) != TIER_EXCLUDE} + pruned = len(requests) - len(keep) + requests = keep + if not requests: + per_kind[kind] = 0.0 + continue kind_started = time.monotonic() batch = getattr(cpg, "run_batch", None) if callable(batch): @@ -338,6 +358,7 @@ def scan_repository( degradations=tuple(degradations), tiers=tuple(tiers), tier_counts={tier: sum(1 for t in tiers if t[3] == tier) for tier in sorted({t[3] for t in tiers})}, + requests_pruned=pruned, ) diff --git a/tests/test_model_scan.py b/tests/test_model_scan.py index 979926d..0b1f94d 100644 --- a/tests/test_model_scan.py +++ b/tests/test_model_scan.py @@ -419,3 +419,56 @@ def test_a_region_cap_that_hides_most_of_a_repository_is_reported() -> None: truncated = [d for d in result.degradations if d.get("reason") == "regions_truncated"] assert truncated, "the cap hid two thirds of the regions and said nothing" assert truncated[0]["examined"] == 10 and truncated[0]["total"] == 30 + + +def test_tier_zero_pairs_are_not_asked_and_nothing_else_is_skipped() -> None: + """Phase 1 of flow-aware-ranking. A pair with no sink of its family in reach cannot yield a flow, so its + taint request is never issued -- and every other pair's request still is.""" + from openultrasast.model.regions import ScanRegion + from openultrasast.model.scan import ScanBudget, scan_repository + + issued: list[dict[str, object]] = [] + + class _Backend: + def available(self) -> bool: + return True + + def build(self, root, *, language=""): # type: ignore[no-untyped-def] + from openultrasast.cpg.backend import CpgResult + + def batch(kind, requests): # type: ignore[no-untyped-def] + if any(p.get("evidenceOnly") == "true" for p in requests.values()): + # Every family but the first has no sink in reach. + return { + rid: [{"kind": "summary", "sinks": 1 if i == 0 else 0, "sourceLocal": True, "familyInRepo": True}] + + ( + [ + { + "kind": "sink", + "sink": "os.system(x)", + "sinkLine": "4", + "sinkMethod": "run", + "sinkArity": 1, + "sinkArg0Literal": False, + "cleansedOnCall": False, + } + ] + if i == 0 + else [] + ) + for i, rid in enumerate(requests) + } + issued.append({"kind": kind, "count": len(requests)}) + return {rid: [] for rid in requests} + + return CpgResult(cpg_path=Path("cpg.bin"), run=lambda q, p: [], run_batch=batch) + + region = ScanRegion( + path="a.py", function="run", language="python", families=("injection", "path", "deserialization"), rank=0.9, source="entry_point" + ) + result = scan_repository(Path("."), [region], backend=_Backend(), budget=ScanBudget(max_model_calls=0)) + + taint = next(i for i in issued if i["kind"] == "taint") + assert taint["count"] == 1, "only the pair with a sink in reach is asked" + assert result.requests_pruned == 2, "and the two tier-0 pairs are counted, not silently dropped" + assert not any(d.get("reason") == "query_failed" for d in result.degradations)